def format_scattered_task(task_name, task_metadata): filtered_shards = [] current_shard = '' min_start = _parse_datetime( task_metadata[0].get('start')) or _parse_datetime( task_metadata[0].get('end')) or offset_aware_now max_end = _parse_datetime(task_metadata[-1].get('end')) execution_events = _get_execution_events(task_metadata) # go through calls in reverse to grab the latest attempt if there are multiple for shard in task_metadata[::-1]: if current_shard != shard.get('shardIndex'): failure_messages = None if shard.get('failures'): failure_messages = [ f.get('message') for f in shard.get('failures') ] filtered_shards.append( Shard(execution_status=task_statuses.cromwell_execution_to_api( shard.get('executionStatus')), start=_parse_datetime(shard.get('start')) or _parse_datetime(shard.get('end')) or offset_aware_now, end=_parse_datetime(shard.get('end')), shard_index=shard.get('shardIndex'), backend_log=shard.get('backendLogs').get('log') if shard.get('backendLogs') else None, call_root=shard.get('callRoot'), operation_id=shard.get('jobId'), attempts=shard.get('attempt'), failure_messages=failure_messages, job_id=shard.get('subWorkflowId'))) if shard.get('start') and min_start > _parse_datetime( shard.get('start')): min_start = _parse_datetime(shard.get('start')) if shard.get('executionStatus') not in [ 'Failed', 'Done', 'Aborted' ]: max_end = None if max_end is not None and max_end < _parse_datetime( shard.get('end')): max_end = _parse_datetime(shard.get('end')) current_shard = shard.get('shardIndex') sorted_shards = sorted(filtered_shards, key=lambda t: t.shard_index) return TaskMetadata( name=remove_workflow_name(task_name), execution_status=_get_scattered_task_status(sorted_shards), attempts=len(sorted_shards), start=min_start, end=max_end, call_root=remove_shard_path(task_metadata[-1].get('callRoot')), shards=sorted_shards, execution_events=execution_events, call_cached=False)
def format_task(task_name, task_metadata): return TaskMetadata( name=remove_workflow_name(task_name), execution_id=task_metadata.get('jobId'), execution_status=task_statuses.cromwell_execution_to_api( task_metadata.get('executionStatus')), start=_parse_datetime(task_metadata.get('start')), end=_parse_datetime(task_metadata.get('end')), stderr=task_metadata.get('stderr'), stdout=task_metadata.get('stdout'), inputs=update_key_names(task_metadata.get('inputs', {})), return_code=task_metadata.get('returnCode'), attempts=task_metadata.get('attempt'), call_root=task_metadata.get('callRoot'), job_id=task_metadata.get('subWorkflowId'))
def _convert_to_attempt(item): attempt = IndividualAttempt( execution_status=task_statuses.cromwell_execution_to_api( item.get('executionStatus')), attempt_number=item.get('attempt'), call_cached=_is_call_cached(item.get('callCaching')), stdout=item.get('stdout'), stderr=item.get('stderr'), call_root=item.get('callRoot'), inputs=item.get('inputs'), outputs=item.get('outputs'), start=_parse_datetime(item.get('start')), end=_parse_datetime(item.get('end'))) if item.get('failures'): attempt.failure_messages = [ f.get('message') for f in item.get('failures') ] return attempt
def _convert_to_attempt(item): attempt = IndividualAttempt( execution_status=task_statuses.cromwell_execution_to_api( item.get('executionStatus')), attempt_number=item.get('attempt'), call_cached=_is_call_cached(item.get('callCaching')), backend_log=item.get('backendLogs').get('log') if item.get('backendLogs') else None, call_root=item.get('callRoot'), operation_id=item.get('jobId'), inputs=item.get('inputs'), outputs=item.get('outputs'), start=_parse_datetime(item.get('start')) or _parse_datetime(item.get('end')) or offset_aware_now, end=_parse_datetime(item.get('end')) or offset_aware_now) if item.get('failures'): attempt.failure_messages = [ f.get('message') for f in item.get('failures') ] return attempt
def format_task(task_name, task_metadata): # check to see if task is scattered if task_metadata[0].get('shardIndex') != -1: return format_scattered_task(task_name, task_metadata) latest_attempt = task_metadata[-1] call_cached = False if latest_attempt.get('callCaching'): call_cached = latest_attempt.get('callCaching') and (_is_call_cached( latest_attempt.get('callCaching'))) execution_events = _get_execution_events(task_metadata) failure_messages = None if latest_attempt.get('failures'): failure_messages = [ f.get('message') for f in latest_attempt.get('failures') ] return TaskMetadata( name=remove_workflow_name(task_name), execution_status=task_statuses.cromwell_execution_to_api( latest_attempt.get('executionStatus')), start=_parse_datetime(latest_attempt.get('start')) or _parse_datetime(latest_attempt.get('end')) or offset_aware_now, end=_parse_datetime(latest_attempt.get('end')), backend_log=latest_attempt.get('backendLogs').get('log') if latest_attempt.get('backendLogs') else None, inputs=update_key_names(latest_attempt.get('inputs', {})), outputs=update_key_names(latest_attempt.get('outputs', {})), return_code=latest_attempt.get('returnCode'), attempts=latest_attempt.get('attempt'), call_root=latest_attempt.get('callRoot'), operation_id=latest_attempt.get('jobId'), job_id=latest_attempt.get('subWorkflowId'), shards=None, call_cached=call_cached, execution_events=execution_events, failure_messages=failure_messages)
def test_unrecognized_task_status_causes_exception(self): with self.assertRaises(ValueError): task_statuses.cromwell_execution_to_api('Not a valid task status')
def test_cromwell_execution_to_api_maps_all_task_execution_statuses_correctly(self): self.assertEqual(task_statuses.cromwell_execution_to_api('NotStarted'), 'Submitted') self.assertEqual(task_statuses.cromwell_execution_to_api('WaitingForQueueSpace'), 'Submitted') self.assertEqual(task_statuses.cromwell_execution_to_api('QueuedInCromwell'), 'Submitted') self.assertEqual(task_statuses.cromwell_execution_to_api('Starting'), 'Submitted') self.assertEqual(task_statuses.cromwell_execution_to_api('Running'), 'Running') self.assertEqual(task_statuses.cromwell_execution_to_api('Aborting'), 'Aborting') self.assertEqual(task_statuses.cromwell_execution_to_api('Unstartable'), 'Failed') self.assertEqual(task_statuses.cromwell_execution_to_api('Aborted'), 'Aborted') self.assertEqual(task_statuses.cromwell_execution_to_api('Bypassed'), 'Submitted') self.assertEqual(task_statuses.cromwell_execution_to_api('RetryableFailure'), 'Failed') self.assertEqual(task_statuses.cromwell_execution_to_api('Failed'), 'Failed') self.assertEqual(task_statuses.cromwell_execution_to_api('Done'), 'Succeeded')