Exemplo n.º 1
0
 def update_atom_details(self, ad):
     """Update a atom detail transactionally."""
     with self._exc_wrapper():
         txn = self._client.transaction()
         ad = self._update_atom_details(ad, txn)
         k_utils.checked_commit(txn)
         return ad
Exemplo n.º 2
0
 def update_flow_details(self, fd):
     """Update a flow detail transactionally."""
     with self._exc_wrapper():
         txn = self._client.transaction()
         fd = self._update_flow_details(fd, txn)
         k_utils.checked_commit(txn)
         return fd
Exemplo n.º 3
0
    def destroy_logbook(self, lb_uuid):
        """Destroy (delete) a log_book transactionally."""

        def _destroy_atom_details(ad_uuid, txn):
            ad_path = paths.join(self.atom_path, ad_uuid)
            if not self._client.exists(ad_path):
                raise exc.NotFound("No atom details found with id: %s"
                                   % ad_uuid)
            txn.delete(ad_path)

        def _destroy_flow_details(fd_uuid, txn):
            fd_path = paths.join(self.flow_path, fd_uuid)
            if not self._client.exists(fd_path):
                raise exc.NotFound("No flow details found with id: %s"
                                   % fd_uuid)
            for ad_uuid in self._client.get_children(fd_path):
                _destroy_atom_details(ad_uuid, txn)
                txn.delete(paths.join(fd_path, ad_uuid))
            txn.delete(fd_path)

        def _destroy_logbook(lb_uuid, txn):
            lb_path = paths.join(self.book_path, lb_uuid)
            if not self._client.exists(lb_path):
                raise exc.NotFound("No logbook found with id: %s" % lb_uuid)
            for fd_uuid in self._client.get_children(lb_path):
                _destroy_flow_details(fd_uuid, txn)
                txn.delete(paths.join(lb_path, fd_uuid))
            txn.delete(lb_path)

        with self._exc_wrapper():
            txn = self._client.transaction()
            _destroy_logbook(lb_uuid, txn)
            k_utils.checked_commit(txn)
Exemplo n.º 4
0
    def clear_all(self, delete_dirs=True):
        """Delete all data transactionally."""
        with self._exc_wrapper():
            txn = self._client.transaction()

            # Delete all data under logbook path.
            for lb_uuid in self._client.get_children(self.book_path):
                lb_path = paths.join(self.book_path, lb_uuid)
                for fd_uuid in self._client.get_children(lb_path):
                    txn.delete(paths.join(lb_path, fd_uuid))
                txn.delete(lb_path)

            # Delete all data under flow detail path.
            for fd_uuid in self._client.get_children(self.flow_path):
                fd_path = paths.join(self.flow_path, fd_uuid)
                for ad_uuid in self._client.get_children(fd_path):
                    txn.delete(paths.join(fd_path, ad_uuid))
                txn.delete(fd_path)

            # Delete all data under atom detail path.
            for ad_uuid in self._client.get_children(self.atom_path):
                ad_path = paths.join(self.atom_path, ad_uuid)
                txn.delete(ad_path)

            # Delete containing directories.
            if delete_dirs:
                txn.delete(self.book_path)
                txn.delete(self.atom_path)
                txn.delete(self.flow_path)

            k_utils.checked_commit(txn)
Exemplo n.º 5
0
    def clear_all(self, delete_dirs=True):
        """Delete all data transactionally."""
        with self._exc_wrapper():
            txn = self._client.transaction()

            # Delete all data under logbook path.
            for lb_uuid in self._client.get_children(self.book_path):
                lb_path = paths.join(self.book_path, lb_uuid)
                for fd_uuid in self._client.get_children(lb_path):
                    txn.delete(paths.join(lb_path, fd_uuid))
                txn.delete(lb_path)

            # Delete all data under flow detail path.
            for fd_uuid in self._client.get_children(self.flow_path):
                fd_path = paths.join(self.flow_path, fd_uuid)
                for ad_uuid in self._client.get_children(fd_path):
                    txn.delete(paths.join(fd_path, ad_uuid))
                txn.delete(fd_path)

            # Delete all data under atom detail path.
            for ad_uuid in self._client.get_children(self.atom_path):
                ad_path = paths.join(self.atom_path, ad_uuid)
                txn.delete(ad_path)

            # Delete containing directories.
            if delete_dirs:
                txn.delete(self.book_path)
                txn.delete(self.atom_path)
                txn.delete(self.flow_path)

            k_utils.checked_commit(txn)
Exemplo n.º 6
0
 def update_flow_details(self, fd):
     """Update a flow detail transactionally."""
     with self._exc_wrapper():
         txn = self._client.transaction()
         fd = self._update_flow_details(fd, txn)
         k_utils.checked_commit(txn)
         return fd
Exemplo n.º 7
0
 def update_atom_details(self, ad):
     """Update a atom detail transactionally."""
     with self._exc_wrapper():
         txn = self._client.transaction()
         ad = self._update_atom_details(ad, txn)
         k_utils.checked_commit(txn)
         return ad
Exemplo n.º 8
0
    def destroy_logbook(self, lb_uuid):
        """Destroy (delete) a log_book transactionally."""
        def _destroy_atom_details(ad_uuid, txn):
            ad_path = paths.join(self.atom_path, ad_uuid)
            if not self._client.exists(ad_path):
                raise exc.NotFound("No atom details found with id: %s" %
                                   ad_uuid)
            txn.delete(ad_path)

        def _destroy_flow_details(fd_uuid, txn):
            fd_path = paths.join(self.flow_path, fd_uuid)
            if not self._client.exists(fd_path):
                raise exc.NotFound("No flow details found with id: %s" %
                                   fd_uuid)
            for ad_uuid in self._client.get_children(fd_path):
                _destroy_atom_details(ad_uuid, txn)
                txn.delete(paths.join(fd_path, ad_uuid))
            txn.delete(fd_path)

        def _destroy_logbook(lb_uuid, txn):
            lb_path = paths.join(self.book_path, lb_uuid)
            if not self._client.exists(lb_path):
                raise exc.NotFound("No logbook found with id: %s" % lb_uuid)
            for fd_uuid in self._client.get_children(lb_path):
                _destroy_flow_details(fd_uuid, txn)
                txn.delete(paths.join(lb_path, fd_uuid))
            txn.delete(lb_path)

        with self._exc_wrapper():
            txn = self._client.transaction()
            _destroy_logbook(lb_uuid, txn)
            k_utils.checked_commit(txn)
Exemplo n.º 9
0
    def save_logbook(self, lb):
        """Save (update) a log_book transactionally."""

        def _create_logbook(lb_path, txn):
            lb_data = lb.to_dict(marshal_time=True)
            txn.create(lb_path, misc.binary_encode(jsonutils.dumps(lb_data)))
            for fd in lb:
                # NOTE(harlowja): create an entry in the logbook path
                # for the provided flow detail so that a reference exists
                # from the logbook to its flow details.
                txn.create(paths.join(lb_path, fd.uuid))
                fd_path = paths.join(self.flow_path, fd.uuid)
                fd_data = jsonutils.dumps(fd.to_dict())
                txn.create(fd_path, misc.binary_encode(fd_data))
                for ad in fd:
                    # NOTE(harlowja): create an entry in the flow detail path
                    # for the provided atom detail so that a reference exists
                    # from the flow detail to its atom details.
                    txn.create(paths.join(fd_path, ad.uuid))
                    ad_path = paths.join(self.atom_path, ad.uuid)
                    ad_data = base._format_atom(ad)
                    txn.create(ad_path,
                               misc.binary_encode(jsonutils.dumps(ad_data)))
            return lb

        def _update_logbook(lb_path, lb_data, txn):
            e_lb = logbook.LogBook.from_dict(misc.decode_json(lb_data),
                                             unmarshal_time=True)
            e_lb = e_lb.merge(lb)
            lb_data = e_lb.to_dict(marshal_time=True)
            txn.set_data(lb_path, misc.binary_encode(jsonutils.dumps(lb_data)))
            for fd in lb:
                fd_path = paths.join(lb_path, fd.uuid)
                if not self._client.exists(fd_path):
                    # NOTE(harlowja): create an entry in the logbook path
                    # for the provided flow detail so that a reference exists
                    # from the logbook to its flow details.
                    txn.create(fd_path)
                e_fd = self._update_flow_details(fd, txn, create_missing=True)
                e_lb.add(e_fd)
            return e_lb

        with self._exc_wrapper():
            txn = self._client.transaction()
            # Determine whether the desired data exists or not.
            lb_path = paths.join(self.book_path, lb.uuid)
            try:
                lb_data, _zstat = self._client.get(lb_path)
            except k_exc.NoNodeError:
                # Create a new logbook since it doesn't exist.
                e_lb = _create_logbook(lb_path, txn)
            else:
                # Otherwise update the existing logbook instead.
                e_lb = _update_logbook(lb_path, lb_data, txn)
            k_utils.checked_commit(txn)
            return e_lb
Exemplo n.º 10
0
    def save_logbook(self, lb):
        """Save (update) a log_book transactionally."""
        def _create_logbook(lb_path, txn):
            lb_data = lb.to_dict(marshal_time=True)
            txn.create(lb_path, misc.binary_encode(jsonutils.dumps(lb_data)))
            for fd in lb:
                # NOTE(harlowja): create an entry in the logbook path
                # for the provided flow detail so that a reference exists
                # from the logbook to its flow details.
                txn.create(paths.join(lb_path, fd.uuid))
                fd_path = paths.join(self.flow_path, fd.uuid)
                fd_data = jsonutils.dumps(fd.to_dict())
                txn.create(fd_path, misc.binary_encode(fd_data))
                for ad in fd:
                    # NOTE(harlowja): create an entry in the flow detail path
                    # for the provided atom detail so that a reference exists
                    # from the flow detail to its atom details.
                    txn.create(paths.join(fd_path, ad.uuid))
                    ad_path = paths.join(self.atom_path, ad.uuid)
                    ad_data = base._format_atom(ad)
                    txn.create(ad_path,
                               misc.binary_encode(jsonutils.dumps(ad_data)))
            return lb

        def _update_logbook(lb_path, lb_data, txn):
            e_lb = logbook.LogBook.from_dict(misc.decode_json(lb_data),
                                             unmarshal_time=True)
            e_lb = e_lb.merge(lb)
            lb_data = e_lb.to_dict(marshal_time=True)
            txn.set_data(lb_path, misc.binary_encode(jsonutils.dumps(lb_data)))
            for fd in lb:
                fd_path = paths.join(lb_path, fd.uuid)
                if not self._client.exists(fd_path):
                    # NOTE(harlowja): create an entry in the logbook path
                    # for the provided flow detail so that a reference exists
                    # from the logbook to its flow details.
                    txn.create(fd_path)
                e_fd = self._update_flow_details(fd, txn, create_missing=True)
                e_lb.add(e_fd)
            return e_lb

        with self._exc_wrapper():
            txn = self._client.transaction()
            # Determine whether the desired data exists or not.
            lb_path = paths.join(self.book_path, lb.uuid)
            try:
                lb_data, _zstat = self._client.get(lb_path)
            except k_exc.NoNodeError:
                # Create a new logbook since it doesn't exist.
                e_lb = _create_logbook(lb_path, txn)
            else:
                # Otherwise update the existing logbook instead.
                e_lb = _update_logbook(lb_path, lb_data, txn)
            k_utils.checked_commit(txn)
            return e_lb
Exemplo n.º 11
0
    def claim(self, job, who):
        def _unclaimable_try_find_owner(cause):
            try:
                owner = self.find_owner(job)
            except Exception:
                owner = None
            if owner:
                message = "Job %s already claimed by '%s'" % (job.uuid, owner)
            else:
                message = "Job %s already claimed" % (job.uuid)
            excp.raise_with_cause(excp.UnclaimableJob, message, cause=cause)

        with self._wrap(job.uuid,
                        job.path,
                        fail_msg_tpl="Claiming failure: %s"):
            # NOTE(harlowja): post as json which will allow for future changes
            # more easily than a raw string/text.
            value = jsonutils.dumps({
                'owner': who,
            })
            # Ensure the target job is still existent (at the right version).
            job_data, job_stat = self._client.get(job.path)
            txn = self._client.transaction()
            # This will abort (and not create the lock) if the job has been
            # removed (somehow...) or updated by someone else to a different
            # version...
            txn.check(job.path, version=job_stat.version)
            txn.create(job.lock_path,
                       value=misc.binary_encode(value),
                       ephemeral=True)
            try:
                kazoo_utils.checked_commit(txn)
            except k_exceptions.NodeExistsError as e:
                _unclaimable_try_find_owner(e)
            except kazoo_utils.KazooTransactionException as e:
                if len(e.failures) < 2:
                    raise
                else:
                    if isinstance(e.failures[0], k_exceptions.NoNodeError):
                        excp.raise_with_cause(
                            excp.NotFound,
                            "Job %s not found to be claimed" % job.uuid,
                            cause=e.failures[0])
                    if isinstance(e.failures[1], k_exceptions.NodeExistsError):
                        _unclaimable_try_find_owner(e.failures[1])
                    else:
                        excp.raise_with_cause(
                            excp.UnclaimableJob,
                            "Job %s claim failed due to transaction"
                            " not succeeding" % (job.uuid),
                            cause=e)
Exemplo n.º 12
0
    def claim(self, job, who):
        def _unclaimable_try_find_owner(cause):
            try:
                owner = self.find_owner(job)
            except Exception:
                owner = None
            if owner:
                message = "Job %s already claimed by '%s'" % (job.uuid, owner)
            else:
                message = "Job %s already claimed" % (job.uuid)
            excp.raise_with_cause(excp.UnclaimableJob,
                                  message, cause=cause)

        with self._wrap(job.uuid, job.path,
                        fail_msg_tpl="Claiming failure: %s"):
            # NOTE(harlowja): post as json which will allow for future changes
            # more easily than a raw string/text.
            value = jsonutils.dumps({
                'owner': who,
            })
            # Ensure the target job is still existent (at the right version).
            job_data, job_stat = self._client.get(job.path)
            txn = self._client.transaction()
            # This will abort (and not create the lock) if the job has been
            # removed (somehow...) or updated by someone else to a different
            # version...
            txn.check(job.path, version=job_stat.version)
            txn.create(job.lock_path, value=misc.binary_encode(value),
                       ephemeral=True)
            try:
                kazoo_utils.checked_commit(txn)
            except k_exceptions.NodeExistsError as e:
                _unclaimable_try_find_owner(e)
            except kazoo_utils.KazooTransactionException as e:
                if len(e.failures) < 2:
                    raise
                else:
                    if isinstance(e.failures[0], k_exceptions.NoNodeError):
                        excp.raise_with_cause(
                            excp.NotFound,
                            "Job %s not found to be claimed" % job.uuid,
                            cause=e.failures[0])
                    if isinstance(e.failures[1], k_exceptions.NodeExistsError):
                        _unclaimable_try_find_owner(e.failures[1])
                    else:
                        excp.raise_with_cause(
                            excp.UnclaimableJob,
                            "Job %s claim failed due to transaction"
                            " not succeeding" % (job.uuid), cause=e)
Exemplo n.º 13
0
 def abandon(self, job, who):
     with self._wrap(job.uuid, job.path, "Abandonment failure: %s"):
         try:
             owner_data = self._get_owner_and_data(job)
             lock_data, lock_stat, data, data_stat = owner_data
         except k_exceptions.NoNodeError:
             raise excp.JobFailure("Can not abandon a job %s"
                                   " which we can not determine"
                                   " the owner of" % (job.uuid))
         if lock_data.get("owner") != who:
             raise excp.JobFailure("Can not abandon a job %s"
                                   " which is not owned by %s"
                                   % (job.uuid, who))
         txn = self._client.transaction()
         txn.delete(job.lock_path, version=lock_stat.version)
         kazoo_utils.checked_commit(txn)
Exemplo n.º 14
0
 def abandon(self, job, who):
     with self._wrap(job.uuid, job.path, "Abandonment failure: %s"):
         try:
             owner_data = self._get_owner_and_data(job)
             lock_data, lock_stat, data, data_stat = owner_data
         except k_exceptions.NoNodeError:
             raise excp.JobFailure("Can not abandon a job %s"
                                   " which we can not determine"
                                   " the owner of" % (job.uuid))
         if lock_data.get("owner") != who:
             raise excp.JobFailure("Can not abandon a job %s"
                                   " which is not owned by %s" %
                                   (job.uuid, who))
         txn = self._client.transaction()
         txn.delete(job.lock_path, version=lock_stat.version)
         kazoo_utils.checked_commit(txn)
Exemplo n.º 15
0
 def consume(self, job, who):
     with self._wrap(job.uuid, job.path,
                     fail_msg_tpl="Consumption failure: %s"):
         try:
             owner_data = self._get_owner_and_data(job)
             lock_data, lock_stat, data, data_stat = owner_data
         except k_exceptions.NoNodeError:
             excp.raise_with_cause(excp.NotFound,
                                   "Can not consume a job %s"
                                   " which we can not determine"
                                   " the owner of" % (job.uuid))
         if lock_data.get("owner") != who:
             raise excp.JobFailure("Can not consume a job %s"
                                   " which is not owned by %s"
                                   % (job.uuid, who))
         txn = self._client.transaction()
         txn.delete(job.lock_path, version=lock_stat.version)
         txn.delete(job.path, version=data_stat.version)
         kazoo_utils.checked_commit(txn)
         self._remove_job(job.path)
Exemplo n.º 16
0
 def consume(self, job, who):
     with self._wrap(job.uuid, job.path,
                     fail_msg_tpl="Consumption failure: %s"):
         try:
             owner_data = self._get_owner_and_data(job)
             lock_data, lock_stat, data, data_stat = owner_data
         except k_exceptions.NoNodeError:
             excp.raise_with_cause(excp.NotFound,
                                   "Can not consume a job %s"
                                   " which we can not determine"
                                   " the owner of" % (job.uuid))
         if lock_data.get("owner") != who:
             raise excp.JobFailure("Can not consume a job %s"
                                   " which is not owned by %s"
                                   % (job.uuid, who))
         txn = self._client.transaction()
         txn.delete(job.lock_path, version=lock_stat.version)
         txn.delete(job.path, version=data_stat.version)
         kazoo_utils.checked_commit(txn)
         self._remove_job(job.path)
Exemplo n.º 17
0
 def trash(self, job, who):
     with self._wrap(job.uuid, job.path, "Trash failure: %s"):
         try:
             owner_data = self._get_owner_and_data(job)
             lock_data, lock_stat, data, data_stat = owner_data
         except k_exceptions.NoNodeError:
             excp.raise_with_cause(excp.JobFailure,
                                   "Can not trash a job %s"
                                   " which we can not determine"
                                   " the owner of" % (job.uuid))
         if lock_data.get("owner") != who:
             raise excp.JobFailure("Can not trash a job %s"
                                   " which is not owned by %s"
                                   % (job.uuid, who))
         trash_path = job.path.replace(self.path, self.trash_path)
         value = misc.binary_encode(jsonutils.dumps(data))
         txn = self._client.transaction()
         txn.create(trash_path, value=value)
         txn.delete(job.lock_path, version=lock_stat.version)
         txn.delete(job.path, version=data_stat.version)
         kazoo_utils.checked_commit(txn)
Exemplo n.º 18
0
 def trash(self, job, who):
     with self._wrap(job.uuid, job.path, fail_msg_tpl="Trash failure: %s"):
         try:
             owner_data = self._get_owner_and_data(job)
             lock_data, lock_stat, data, data_stat = owner_data
         except k_exceptions.NoNodeError:
             excp.raise_with_cause(
                 excp.NotFound, "Can not trash a job %s"
                 " which we can not determine"
                 " the owner of" % (job.uuid))
         if lock_data.get("owner") != who:
             raise excp.JobFailure("Can not trash a job %s"
                                   " which is not owned by %s" %
                                   (job.uuid, who))
         trash_path = job.path.replace(self.path, self.trash_path)
         value = misc.binary_encode(jsonutils.dumps(data))
         txn = self._client.transaction()
         txn.create(trash_path, value=value)
         txn.delete(job.lock_path, version=lock_stat.version)
         txn.delete(job.path, version=data_stat.version)
         kazoo_utils.checked_commit(txn)
Exemplo n.º 19
0
 def _transaction(self):
     transaction = self._client.transaction()
     with self._exc_wrapper():
         yield transaction
         k_utils.checked_commit(transaction)
Exemplo n.º 20
0
 def _transaction(self):
     transaction = self._client.transaction()
     with self._exc_wrapper():
         yield transaction
         k_utils.checked_commit(transaction)