Exemplo n.º 1
0
 def _insert_token(self, handler):
     request = ModifyRequest()
     token = Token(name='/some_other_dir/some_token', data='some data')
     request.updates = [token]
     response = handler.modify(request)
     self.assertEqual(1, len(response.updates))
     return response.updates[0]
Exemplo n.º 2
0
 def _insert_token(self, handler):
     request = ModifyRequest()
     token = Token(name='/some_other_dir/some_token', data='some data')
     request.updates = [token]
     response = handler.modify(request)
     self.assertEqual(1, len(response.updates))
     return response.updates[0]
Exemplo n.º 3
0
    def test_recursive(self):
        Options = collections.namedtuple('args',
                                         'recursive force command_args')
        options = Options(recursive=True, force=True,
                          command_args=['/some_path'])
        command = Rm()
        command.prepare(options)

        client = mock.Mock()
        token = Token(version=10,
                      name='/some_path/some_token',
                      owner='some_owner',
                      expirationTime=10,
                      data='some_data')
        query_response = QueryResponse(tokens=[[token]])
        client.query.return_value = query_response

        modify_response = ModifyResponse()
        client.modify.return_value = modify_response

        output = command.execute(client, None)

        query = Query(namePrefix='/some_path')
        query_request = QueryRequest(queries=[query])
        client.query.assert_called_once_with(query_request)

        modify_request = ModifyRequest(deletes=[token])
        client.modify.assert_called_once_with(modify_request)

        self.assertEqual('removed 1 token(s)\n', output)
Exemplo n.º 4
0
    def _post_signal_tokens(self):
        """Add some signal tokens to the master."""
        request = ModifyRequest(updates=[])

        signal = Signal(action=Signal.EXIT)
        name = Name(signal='exit')
        signal_token = Token(name=name.get_signal_token_name())
        signal_token.data = pickle.dumps(signal)
        request.updates.append(signal_token)

        signal = Signal(action=Signal.DRAIN)
        name.signal = 'drain'
        name.workflow = 'some_workflow'
        signal_token = Token(name=name.get_signal_token_name())
        signal_token.data = pickle.dumps(signal)
        request.updates.append(signal_token)

        name.instance = '123'
        signal_token = Token(name=name.get_signal_token_name())
        signal_token.data = pickle.dumps(signal)
        request.updates.append(signal_token)

        signal = Signal(action=Signal.ABORT)
        name.signal = 'abort'
        signal_token = Token(name=name.get_signal_token_name())
        signal_token.data = pickle.dumps(signal)
        request.updates.append(signal_token)

        client = self._factory.get_client()
        client.modify(request)
Exemplo n.º 5
0
 def test_modity_empty(self):
     request = ModifyRequest()
     transaction = ModifyTransaction()
     # Make sure that prepare and commit do not throw an exception.
     transaction.prepare(request)
     transaction.commit(self._trie, self._get_blessed_version(),
                        self._store)
Exemplo n.º 6
0
    def set_action(self, action):
        """Send a signal with a specific action to the master.

        Local signal store gets updated with the new action if it is
        successfully submitted to the master.  If the communication with the
        master fails, locally stored signals get refreshed.

        Args:
            action: The action to set.
        """
        attributes = {}
        if action == Signal.ABORT:
            attributes[Signal.TIMESTAMP_ATTR] = time.time()
        elif action == Signal.EXIT:
            attributes[Signal.GENERATION_ATTR] = PinballConfig.GENERATION
        signal = self._signals.get(action)
        if signal and signal.attributes == attributes:
            return
        # A signal with the same action but different data may already exist
        # in the master.
        signal_token = self._get_signal_token(action)
        if not signal_token:
            name = Name(workflow=self._workflow,
                        instance=self._instance,
                        signal=Signal.action_to_string(action))
            signal_token = Token(name=name.get_signal_token_name())
        signal = Signal(action, attributes)
        signal_token.data = pickle.dumps(signal)
        request = ModifyRequest(updates=[signal_token])
        if self._send_request(request):
            self._signals[action] = signal
Exemplo n.º 7
0
    def set_attribute_if_missing(self, action, attribute, value):
        """Set an attribute value unless that attribute is already set.

        Args:
            action: The action whose attribute should be set.
            attribute: The attribute to set.
            value: The attribute value to set.
        Returns:
            True iff the attribute value was set.  Return False if the signal
            is not set.
        """
        if self.get_attribute(action, attribute) is not None:
            return False
        signal_token = self._get_signal_token(action)
        if not signal_token:
            return False
        signal = pickle.loads(signal_token.data)
        self._signals[action] = signal
        if self.get_attribute(action, attribute) is not None:
            return False
        signal.attributes[attribute] = value
        signal_token.data = pickle.dumps(signal)
        request = ModifyRequest(updates=[signal_token])
        if self._send_request(request):
            self._signals[action] = signal
            return True
        return False
Exemplo n.º 8
0
    def test_update(self):
        Options = collections.namedtuple('args', 'name version owner '
                                         'expiration_time priority data '
                                         'command_args')
        options = Options(name='/some_path/some_token',
                          version=10,
                          owner='some_other_owner',
                          expiration_time=100,
                          priority=10,
                          data='some_other_data',
                          command_args=None)
        command = Update()
        command.prepare(options)
        client = mock.Mock()
        output_token = Token(version=11,
                             name='/some_path/some_token',
                             owner='some_other_owner',
                             expirationTime=100,
                             priority=10,
                             data='some_other_data')
        response = ModifyResponse(updates=[output_token])
        client.modify.return_value = response

        output = command.execute(client, None)

        input_token = Token(version=10,
                            name='/some_path/some_token',
                            owner='some_other_owner',
                            expirationTime=100,
                            priority=10,
                            data='some_other_data')
        request = ModifyRequest(updates=[input_token])
        client.modify.assert_called_once_with(request)
        self.assertEqual('updated %s\nupdated 1 token\n' % str(output_token),
                         output)
Exemplo n.º 9
0
    def _update_owned_job_token(self):
        """Update owned job token in the master.

        Returns:
            True if the update was successful, otherwise False.
        """
        assert self._owned_job_token
        request = ModifyRequest()
        request.updates = [self._owned_job_token]
        try:
            response = self._client.modify(request)
        except TokenMasterException:
            LOG.exception('error sending request %s', request)
            return False
        assert len(response.updates) == 1
        self._owned_job_token = response.updates[0]
        return True
Exemplo n.º 10
0
    def _update_owned_job_token(self):
        """Update owned job token in the master.

        Returns:
            True if the update was successful, otherwise False.
        """
        assert self._owned_job_token
        request = ModifyRequest()
        request.updates = [self._owned_job_token]
        try:
            response = self._client.modify(request)
        except TokenMasterException:
            LOG.exception('error sending request %s', request)
            return False
        assert len(response.updates) == 1
        self._owned_job_token = response.updates[0]
        return True
Exemplo n.º 11
0
    def run(self, emailer, store):
        if not self._check_workflow_instances(emailer, self.workflow, store):
            LOG.warn('too many instances running for workflow %s', self.workflow)
            return None

        config_parser = load_path(PinballConfig.PARSER)(self.parser_params)
        workflow_tokens = config_parser.get_workflow_tokens(self.workflow)
        if not workflow_tokens:
            LOG.error('workflow %s not found', self.workflow)
            return None
        result = ModifyRequest()
        result.updates = workflow_tokens
        assert result.updates
        token = result.updates[0]
        name = Name.from_job_token_name(token.name)
        if not name.instance:
            name = Name.from_event_token_name(token.name)
        LOG.info('exporting workflow %s instance %s.  Its tokens are under %s',
                 name.workflow, name.instance, name.get_instance_prefix())
        return result
Exemplo n.º 12
0
 def _post_workflow_start_event_token(self):
     name = Name(workflow='some_workflow',
                 instance='12345',
                 job='parent_job',
                 input_name=Name.WORKFLOW_START_INPUT,
                 event='workflow_start_event')
     event = Event(creator='SimpleWorkflowTest')
     token = Token(name=name.get_event_token_name(),
                   data=pickle.dumps(event))
     request = ModifyRequest(updates=[token])
     self._client.modify(request)
Exemplo n.º 13
0
    def run(self, emailer, store):
        if not self._check_workflow_instances(emailer, self.workflow, store):
            LOG.warn('too many instances running for workflow %s',
                     self.workflow)
            return None

        config_parser = load_path(PinballConfig.PARSER)(self.parser_params)
        workflow_tokens = config_parser.get_workflow_tokens(self.workflow)
        if not workflow_tokens:
            LOG.error('workflow %s not found', self.workflow)
            return None
        result = ModifyRequest()
        result.updates = workflow_tokens
        assert result.updates
        token = result.updates[0]
        name = Name.from_job_token_name(token.name)
        if not name.instance:
            name = Name.from_event_token_name(token.name)
        LOG.info('exporting workflow %s instance %s.  Its tokens are under %s',
                 name.workflow, name.instance, name.get_instance_prefix())
        return result
Exemplo n.º 14
0
    def _keep_job_token_in_runnable(self, job):
        """Keep the owned job token in the runnable group.

        Refresh the job token data field with the provided job object, release
        the ownership of the token, and return it to the runnable group.

        Args:
            job: The job that should be stored in the data field of the job
                token.
        """
        assert self._owned_job_token
        request = ModifyRequest()
        self._owned_job_token.data = pickle.dumps(job)
        retry_delay_sec = job.retry_delay_sec
        if retry_delay_sec > 0:
            self._owned_job_token.expirationTime = (time.time() +
                                                    retry_delay_sec)
        else:
            self._unown(self._owned_job_token)
        request.updates = [self._owned_job_token]
        self._send_request(request)
Exemplo n.º 15
0
    def _keep_job_token_in_runnable(self, job):
        """Keep the owned job token in the runnable group.

        Refresh the job token data field with the provided job object, release
        the ownership of the token, and return it to the runnable group.

        Args:
            job: The job that should be stored in the data field of the job
                token.
        """
        assert self._owned_job_token
        request = ModifyRequest()
        self._owned_job_token.data = pickle.dumps(job)
        retry_delay_sec = job.retry_delay_sec
        if retry_delay_sec > 0:
            self._owned_job_token.expirationTime = (time.time() +
                                                    retry_delay_sec)
        else:
            self._unown(self._owned_job_token)
        request.updates = [self._owned_job_token]
        self._send_request(request)
Exemplo n.º 16
0
    def remove_action(self, action):
        """Remove signal with a given action from the master.

        Args:
            action: The action to remove.
        """
        if not self.is_signal_present(action):
            return
        signal_token = self._get_signal_token(action)
        if signal_token:
            request = ModifyRequest(deletes=[signal_token])
            self._client.modify(request)
        del self._signals[action]
Exemplo n.º 17
0
 def execute(self, client, store):
     output = ''
     tokens = _get_tokens(self._prefix, self._recursive, client)
     deleted = 0
     if not tokens:
         output += 'no tokens found\n'
     else:
         print 'removing:'
         for token in tokens:
             print '\t%s' % token.name
         if self._force or confirm('remove %d tokens' % len(tokens)):
             request = ModifyRequest(deletes=tokens)
             client.modify(request)
             deleted = len(tokens)
     output += 'removed %d token(s)\n' % deleted
     return output
Exemplo n.º 18
0
 def test_modify_deletes(self):
     request = ModifyRequest(deletes=[])
     n_tokens_before = len(self._trie)
     some_token = copy.copy(self._trie['/some_dir/some_token_0'])
     request.deletes.append(some_token)
     some_other_token = copy.copy(
         self._trie['/some_dir/some_token_0/some_other_token_0'])
     request.deletes.append(some_other_token)
     transaction = ModifyTransaction()
     transaction.prepare(request)
     transaction.commit(self._trie, self._get_blessed_version(),
                        self._store)
     n_tokens_after = len(self._trie)
     # We deleted two things from self._trie.
     self.assertEqual(n_tokens_before - 2, n_tokens_after)
     self._check_version_uniqueness()
Exemplo n.º 19
0
 def test_modify_no_name_change(self):
     """Do not allow changing token names."""
     request = ModifyRequest(updates=[])
     # pickle gets maximum recursion depth exceeded when traversing
     # the trie, probably a bug in pickle. Setting the recursion limit
     # to a high number fixes it.
     sys.setrecursionlimit(10000)
     trie_before = pickle.dumps(self._trie)
     token = copy.copy(self._trie['/some_dir/some_token_0'])
     token.name = '/some_other_dir/some_token_0'
     request.updates.append(token)
     transaction = ModifyTransaction()
     transaction.prepare(request)
     self.assertRaises(TokenMasterException, transaction.commit, self._trie,
                       self._get_blessed_version(), self._store)
     trie_after = pickle.dumps(self._trie)
     self.assertEqual(trie_before, trie_after)
Exemplo n.º 20
0
 def execute(self, client, store):
     output = ''
     token = Token(name=self._name,
                   version=self._version,
                   owner=self._owner,
                   expirationTime=self._expiration_time,
                   priority=self._priority,
                   data=self._data)
     request = ModifyRequest(updates=[token])
     response = client.modify(request)
     assert len(response.updates) == 1
     if token.version is None:
         action = 'inserted'
     else:
         action = 'updated'
     output += '%s %s\n' % (action, str(response.updates[0]))
     output += 'updated 1 token\n'
     return output
Exemplo n.º 21
0
    def test_modify_deletes_and_updates(self):
        """Updates and deletes in a single request."""
        request = ModifyRequest(updates=[], deletes=[])
        n_tokens_before = len(self._trie)
        delete_token = copy.copy(self._trie['/some_dir/some_token_0'])
        request.deletes.append(delete_token)
        update_token = copy.copy(self._trie['/some_dir/some_token_1'])
        update_token.data = 'some other data'
        request.updates.append(update_token)
        transaction = ModifyTransaction()
        transaction.prepare(request)
        response = transaction.commit(self._trie, self._get_blessed_version(),
                                      self._store)

        self.assertEqual(1, len(response.updates))
        n_tokens_after = len(self._trie)
        self.assertEqual(n_tokens_before - 1, n_tokens_after)
        self._check_version_uniqueness()
Exemplo n.º 22
0
 def _update_tokens(self):
     """Update tokens modified during schedule execution in the master.
     """
     assert self._owned_schedule_token
     if not self._request:
         self._request = ModifyRequest()
     if not self._request.updates:
         self._request.updates = []
     self._request.updates.append(self._owned_schedule_token)
     schedule = pickle.loads(self._owned_schedule_token.data)
     if schedule.workflow == 'experiments':
         LOG.info('updating tokens for workflow experiments %s',
                  self._request)
     try:
         self._client.modify(self._request)
     except TokenMasterException:
         LOG.exception('')
     finally:
         self._owned_schedule_token = None
         self._request = None
Exemplo n.º 23
0
    def _move_job_token_to_runnable(self, job_token, triggering_event_tokens):
        """Move a job token to the runnable branch of the token tree.

        Token tree is the global, hierarchically structured token namespace.
        Args:
            job_token: The job token to make runnable.
            triggering_event_tokens: The list of events used to trigger the
                job.  These events will be removed from the master in the same
                call to that makes the job token runnable.
        Returns:
            True on success, otherwise False.
        """
        name = Name.from_job_token_name(job_token.name)
        name.job_state = Name.RUNNABLE_STATE
        job = pickle.loads(job_token.data)
        Worker._add_events_to_job(job, triggering_event_tokens)
        runnable_job_token = Token(name=name.get_job_token_name(),
                                   priority=job_token.priority,
                                   data=pickle.dumps(job))
        request = ModifyRequest(updates=[runnable_job_token],
                                deletes=triggering_event_tokens + [job_token])
        return self._send_request(request)
Exemplo n.º 24
0
    def _move_job_token_to_waiting(self, job, succeeded):
        """Move the owned job token to the waiting group.

        If the job succeeded, also post events to job outputs.  If the job
        failed or it is the final job (a job with no outputs),  post an archive
        signal to finish the workflow.

        Args:
            job: The job that should be stored in the data field of the waiting
                job token.
            succeeded: True if the job succeeded, otherwise False.
        """
        assert self._owned_job_token
        name = Name.from_job_token_name(self._owned_job_token.name)
        name.job_state = Name.WAITING_STATE
        waiting_job_token = Token(name=name.get_job_token_name(),
                                  priority=self._owned_job_token.priority,
                                  data=pickle.dumps(job))
        request = ModifyRequest(deletes=[self._owned_job_token],
                                updates=[waiting_job_token])
        if succeeded:
            request.updates.extend(self._get_output_event_tokens(job))
        if not job.outputs or not succeeded:
            # This is either the only job in the workflow with no outputs or a
            # failed job.  In either case, the workflow is done.
            signaller = Signaller(self._client,
                                  workflow=name.workflow,
                                  instance=name.instance)
            if not signaller.is_action_set(Signal.ARCHIVE):
                signal_name = Name(workflow=name.workflow,
                                   instance=name.instance,
                                   signal=Signal.action_to_string(
                                       Signal.ARCHIVE))
                signal = Signal(Signal.ARCHIVE)
                signal_token = Token(name=signal_name.get_signal_token_name())
                signal_token.data = pickle.dumps(signal)
                request.updates.append(signal_token)
        self._send_request(request)
Exemplo n.º 25
0
    def test_modify_updates(self):
        request = ModifyRequest(updates=[])
        n_tokens_before = len(self._trie)
        token = copy.copy(self._trie['/some_dir/some_token_0'])
        token.data = 'some other data'
        request.updates.append(token)
        new_token = Token(name='/some_other_dir/some_token', data='some data')
        request.updates.append(new_token)
        transaction = ModifyTransaction()
        transaction.prepare(request)
        response = transaction.commit(self._trie, self._get_blessed_version(),
                                      self._store)

        self.assertEqual(2, len(response.updates))
        self.assertNotEqual(token.version, response.updates[0].version)
        self.assertEqual(token.name, response.updates[0].name)
        self.assertEqual(token.data, response.updates[0].data)
        self.assertLess(0, response.updates[1].version)
        self.assertEqual(new_token.name, response.updates[1].name)
        self.assertEqual(new_token.data, response.updates[1].data)
        n_tokens_after = len(self._trie)
        self.assertEqual(n_tokens_before + 1, n_tokens_after)
        self._check_version_uniqueness()
Exemplo n.º 26
0
 def _post_schedule_token(self):
     """Add schedule token to the master."""
     request = ModifyRequest()
     request.updates = [SchedulerTestCase._get_schedule_token()]
     self._client.modify(request)
Exemplo n.º 27
0
 def run(self, emailer, store):
     return ModifyRequest(updates=[Token()])
Exemplo n.º 28
0
 def _post_schedule_token(self):
     """Add schedule token to the master."""
     request = ModifyRequest()
     request.updates = [SchedulerTestCase._get_schedule_token()]
     self._client.modify(request)
Exemplo n.º 29
0
 def _post_job_tokens(self):
     """Add waiting job tokens to the master."""
     request = ModifyRequest(updates=[])
     request.updates.append(self._get_parent_job_token())
     request.updates.append(self._get_child_job_token())
     self._client.modify(request)