def _own_schedule_token_list(self):
"""Attempt to own some schedule tokens.
Only unowned tokens will be considered. Unowned schedules are ready to
run. The ownership of the qualifying job token lasts for a limited
time so it has to be periodically renewed if the schedule takes longer
than that to run.
"""
assert not self._owned_schedule_token
query = Query()
query.namePrefix = Name.SCHEDULE_PREFIX
query.maxTokens = self._SCHEDULE_GANG_SIZE
request = QueryAndOwnRequest()
request.query = query
request.expirationTime = int(time.time()) + Scheduler._LEASE_TIME_SEC
request.owner = self._name
try:
response = self._client.query_and_own(request)
if response.tokens:
assert len(response.tokens) <= self._SCHEDULE_GANG_SIZE
self._owned_schedule_token_list = [
token for token in response.tokens if token
]
LOG.info("got %d schedule token(s) from master.",
len(self._owned_schedule_token_list))
except TokenMasterException:
LOG.exception('')
def _query_and_own_runnable_job_token(self, workflow, instance):
"""Attempt to own a runnable job token from a given workflow instance.
Try to own a runnable job token in a given workflow instance. The
ownership of the qualifying job token lasts for a limited time so it
has to be periodically renewed.
Args:
workflow: The name of the workflow whose jobs should be considered.
instance: The workflow instance whose jobs should be considered.
"""
assert not self._owned_job_token
name = Name(workflow=workflow,
instance=instance,
job_state=Name.RUNNABLE_STATE)
query = Query()
query.namePrefix = name.get_job_state_prefix()
query.maxTokens = 1
request = QueryAndOwnRequest()
request.query = query
request.expirationTime = time.time() + Worker._LEASE_TIME_SEC
request.owner = self._name
try:
response = self._client.query_and_own(request)
if response.tokens:
assert len(response.tokens) == 1
self._owned_job_token = response.tokens[0]
except TokenMasterException:
LOG.exception('error sending request %s', request)
def _query_and_own_runnable_job_token(self, workflow, instance):
"""Attempt to own a runnable job token from a given workflow instance.
Try to own a runnable job token in a given workflow instance. The
ownership of the qualifying job token lasts for a limited time so it
has to be periodically renewed.
Args:
workflow: The name of the workflow whose jobs should be considered.
instance: The workflow instance whose jobs should be considered.
"""
assert not self._owned_job_token
name = Name(workflow=workflow,
instance=instance,
job_state=Name.RUNNABLE_STATE)
query = Query()
query.namePrefix = name.get_job_state_prefix()
query.maxTokens = 1
request = QueryAndOwnRequest()
request.query = query
request.expirationTime = time.time() + Worker._LEASE_TIME_SEC
request.owner = self._name
try:
response = self._client.query_and_own(request)
if response.tokens:
assert len(response.tokens) == 1
self._owned_job_token = response.tokens[0]
except TokenMasterException:
LOG.exception('error sending request %s', request)
def test_query_and_own(self):
some_token = self._trie['/some_dir/some_token_0']
some_token.owner = 'some_owner'
some_token.expirationTime = 10 # in the past
some_token = self._trie['/some_dir/some_token_1']
some_token.owner = 'some_owner'
some_token.expirationTime = sys.maxint # in the future
some_query = Query()
some_query.namePrefix = ''
some_query.maxTokens = 200
request = QueryAndOwnRequest()
request.owner = 'some_other_owner'
request.expirationTime = sys.maxint
request.query = some_query
transaction = QueryAndOwnTransaction()
transaction.prepare(request)
response = transaction.commit(self._trie, self._get_blessed_version(),
self._store)
# Should have owned all tokens but two: the blessed version and the one
# token that is already owned.
self.assertEqual(len(self._trie) - 2, len(response.tokens))
for token in response.tokens:
self.assertEquals('some_other_owner', token.owner)
self.assertEquals(sys.maxint, token.expirationTime)
def test_query_and_own(self):
some_token = self._trie['/some_dir/some_token_0']
some_token.owner = 'some_owner'
some_token.expirationTime = 10 # in the past
some_token = self._trie['/some_dir/some_token_1']
some_token.owner = 'some_owner'
some_token.expirationTime = sys.maxint # in the future
some_query = Query()
some_query.namePrefix = ''
some_query.maxTokens = 200
request = QueryAndOwnRequest()
request.owner = 'some_other_owner'
request.expirationTime = sys.maxint
request.query = some_query
transaction = QueryAndOwnTransaction()
transaction.prepare(request)
response = transaction.commit(self._trie,
self._get_blessed_version(),
self._store)
# Should have owned all tokens but two: the blessed version and the one
# token that is already owned.
self.assertEqual(len(self._trie) - 2, len(response.tokens))
for token in response.tokens:
self.assertEquals('some_other_owner', token.owner)
self.assertEquals(sys.maxint, token.expirationTime)
def _own_schedule_token_list(self):
"""Attempt to own some schedule tokens.
Only unowned tokens will be considered. Unowned schedules are ready to
run. The ownership of the qualifying job token lasts for a limited
time so it has to be periodically renewed if the schedule takes longer
than that to run.
"""
assert not self._owned_schedule_token
query = Query()
query.namePrefix = Name.SCHEDULE_PREFIX
query.maxTokens = self._SCHEDULE_GANG_SIZE
request = QueryAndOwnRequest()
request.query = query
request.expirationTime = int(time.time()) + Scheduler._LEASE_TIME_SEC
request.owner = self._name
try:
response = self._client.query_and_own(request)
if response.tokens:
assert len(response.tokens) <= self._SCHEDULE_GANG_SIZE
self._owned_schedule_token_list = [token for token in response.tokens if token]
LOG.info(
"got %d schedule token(s) from master.",
len(self._owned_schedule_token_list)
)
except TokenMasterException:
LOG.exception('')
def test_query(self):
query = Query()
query.namePrefix = ''
query.maxTokens = 10
request = QueryRequest()
request.queries = [query]
handler = MasterHandler(EphemeralStore())
response = handler.query(request)
self.assertEqual(1, len(response.tokens))
def test_query(self):
query = Query()
query.namePrefix = ''
query.maxTokens = 10
request = QueryRequest()
request.queries = [query]
handler = MasterHandler(EphemeralStore())
response = handler.query(request)
self.assertEqual(1, len(response.tokens))
def test_query_and_own(self):
query = Query()
query.namePrefix = ''
query.maxTokens = 10
request = QueryAndOwnRequest()
request.owner = 'some_owner'
request.expirationTime = sys.maxint
request.query = query
handler = MasterHandler(EphemeralStore())
response = handler.query_and_own(request)
self.assertEqual(0, len(response.tokens))
def test_query_and_own(self):
query = Query()
query.namePrefix = ''
query.maxTokens = 10
request = QueryAndOwnRequest()
request.owner = 'some_owner'
request.expirationTime = sys.maxint
request.query = query
handler = MasterHandler(EphemeralStore())
response = handler.query_and_own(request)
self.assertEqual(0, len(response.tokens))
def test_query(self):
some_query = Query()
some_query.namePrefix = '/some_dir'
some_query.maxTokens = 10
some_other_query = Query()
some_other_query.namePrefix = '/some_dir/some_token_0'
some_other_query.maxTokens = 100
request = QueryRequest()
request.queries = [some_query, some_other_query]
transaction = QueryTransaction()
transaction.prepare(request)
response = transaction.commit(self._trie, self._get_blessed_version(),
self._store)
self.assertEqual(2, len(response.tokens))
self.assertEqual(10, len(response.tokens[0]))
for token in response.tokens[0]:
self.assertTrue(token.name.startswith('/some_dir'))
self.assertEqual(9, token.priority)
self.assertEqual(11, len(response.tokens[1]))
for token in response.tokens[1]:
self.assertTrue(token.name.startswith('/some_dir/some_token_0'))
def test_query(self):
some_query = Query()
some_query.namePrefix = '/some_dir'
some_query.maxTokens = 10
some_other_query = Query()
some_other_query.namePrefix = '/some_dir/some_token_0'
some_other_query.maxTokens = 100
request = QueryRequest()
request.queries = [some_query, some_other_query]
transaction = QueryTransaction()
transaction.prepare(request)
response = transaction.commit(self._trie,
self._get_blessed_version(),
self._store)
self.assertEqual(2, len(response.tokens))
self.assertEqual(10, len(response.tokens[0]))
for token in response.tokens[0]:
self.assertTrue(token.name.startswith('/some_dir'))
self.assertEqual(9, token.priority)
self.assertEqual(11, len(response.tokens[1]))
for token in response.tokens[1]:
self.assertTrue(token.name.startswith('/some_dir/some_token_0'))
def _make_job_runnable(self, job_token):
"""Attempt to make a job runnable.
Query event tokens in job inputs. If a combination of triggering
events exist, remove those events and make the job runnable.
Otherwise, do nothing.
Args:
job_token: The job token to make runnable.
Returns:
True if there were no errors during communication with the master,
otherwise False.
"""
job = pickle.loads(job_token.data)
name = Name.from_job_token_name(job_token.name)
request = QueryRequest(queries=[])
# TODO(pawel): handle jobs with no dependencies
assert job.inputs
for input_name in job.inputs:
prefix = Name()
prefix.workflow = name.workflow
prefix.instance = name.instance
prefix.job = name.job
prefix.input = input_name
query = Query()
query.namePrefix = prefix.get_input_prefix()
query.maxTokens = 1
request.queries.append(query)
try:
response = self._client.query(request)
except TokenMasterException:
# TODO(pawel): add a retry count and fail if a limit is reached.
LOG.exception('error sending request %s', request)
return False
triggering_events = Worker._get_triggering_events(response.tokens)
if triggering_events:
return self._move_job_token_to_runnable(job_token,
triggering_events)
return True
def _make_job_runnable(self, job_token):
"""Attempt to make a job runnable.
Query event tokens in job inputs. If a combination of triggering
events exist, remove those events and make the job runnable.
Otherwise, do nothing.
Args:
job_token: The job token to make runnable.
Returns:
True if there were no errors during communication with the master,
otherwise False.
"""
job = pickle.loads(job_token.data)
name = Name.from_job_token_name(job_token.name)
request = QueryRequest(queries=[])
# TODO(pawel): handle jobs with no dependencies
assert job.inputs
for input_name in job.inputs:
prefix = Name()
prefix.workflow = name.workflow
prefix.instance = name.instance
prefix.job = name.job
prefix.input = input_name
query = Query()
query.namePrefix = prefix.get_input_prefix()
query.maxTokens = 1
request.queries.append(query)
try:
response = self._client.query(request)
except TokenMasterException:
# TODO(pawel): add a retry count and fail if a limit is reached.
LOG.exception('error sending request %s', request)
return False
triggering_events = Worker._get_triggering_events(response.tokens)
if triggering_events:
return self._move_job_token_to_runnable(job_token,
triggering_events)
return True
