def test_super_trigger_consume_subscription(self): subscription = self.dart.await_subscription_generation( self.subscription.id) self.assertEqual(subscription.data.state, SubscriptionState.ACTIVE) tr_args = { 'subscription_id': self.subscription.id, 'unconsumed_data_size_in_bytes': 49524 } tr = Trigger(data=TriggerData('test-trigger', 'subscription_batch', None, tr_args, TriggerState.ACTIVE)) self.trigger = self.dart.save_trigger(tr) st_args = { 'fire_after': 'ALL', 'completed_trigger_ids': [self.trigger.id] } st = Trigger( data=TriggerData('test-super-trigger', 'super', [self.workflow.id], st_args, TriggerState.ACTIVE)) super_trigger = self.dart.save_trigger(st) wf_instances = self.dart.await_workflow_completion(self.workflow.id, num_instances=3) for wfi in wf_instances: self.assertEqual(wfi.data.state, WorkflowInstanceState.COMPLETED) stats = self.dart.get_subscription_element_stats(self.subscription.id) ses = SubscriptionElementStats(SubscriptionElementState.CONSUMED, 3, 152875004 + 834620 + 49524) self.assertEqual([s.to_dict() for s in stats], [ses.to_dict()]) self.dart.delete_trigger(super_trigger.id) self.dart.delete_trigger(self.trigger.id)
def test_trigger_schema(self): args = {'completed_workflow_id': 'ABC123'} state = None tr = Trigger(data=TriggerData('test-trigger', 'workflow_completion', ['ABC123'], args, state=state)) obj_before = tr.to_dict() tr = default_and_validate(tr, trigger_schema(workflow_completion_trigger.params_json_schema)) # state should be defaulted to INACTIVE self.assertNotEqual(obj_before, tr.to_dict())
def setUp(self): dart = Dart(host='localhost', port=5000) """ :type dart: dart.client.python.dart_client.Dart """ self.dart = dart dst_args = {'action_sleep_time_in_seconds': 0} dst0 = Datastore(data=DatastoreData('test-datastore0', 'no_op_engine', args=dst_args, state=DatastoreState.TEMPLATE)) self.datastore0 = self.dart.save_datastore(dst0) dst1 = Datastore(data=DatastoreData('test-datastore1', 'no_op_engine', args=dst_args, state=DatastoreState.TEMPLATE)) self.datastore1 = self.dart.save_datastore(dst1) wf0 = Workflow(data=WorkflowData( 'test-workflow0', self.datastore0.id, state=WorkflowState.ACTIVE)) self.workflow0 = self.dart.save_workflow(wf0, self.datastore0.id) wf1 = Workflow(data=WorkflowData( 'test-workflow1', self.datastore1.id, state=WorkflowState.ACTIVE)) self.workflow1 = self.dart.save_workflow(wf1, self.datastore1.id) a00 = Action(data=ActionData(NoOpActionTypes.action_that_succeeds.name, NoOpActionTypes.action_that_succeeds.name, state=ActionState.TEMPLATE)) a01 = Action(data=ActionData(NoOpActionTypes.action_that_succeeds.name, NoOpActionTypes.action_that_succeeds.name, state=ActionState.TEMPLATE)) self.action00, self.action01 = self.dart.save_actions( [a00, a01], workflow_id=self.workflow0.id) a10 = Action(data=ActionData(NoOpActionTypes.action_that_succeeds.name, NoOpActionTypes.action_that_succeeds.name, state=ActionState.TEMPLATE)) a11 = Action(data=ActionData(NoOpActionTypes.action_that_succeeds.name, NoOpActionTypes.action_that_succeeds.name, state=ActionState.TEMPLATE)) self.action10, self.action11 = self.dart.save_actions( [a10, a11], workflow_id=self.workflow1.id) tr_args = {'completed_workflow_id': self.workflow0.id} tr = Trigger(data=TriggerData('test-trigger', 'workflow_completion', None, tr_args, TriggerState.ACTIVE)) self.trigger = self.dart.save_trigger(tr) st_args = { 'fire_after': 'ALL', 'completed_trigger_ids': [self.trigger.id] } st = Trigger(data=TriggerData('test-super-trigger', 'super', [self.workflow1.id], st_args, TriggerState.ACTIVE)) self.super_trigger = self.dart.save_trigger(st)
def test_trigger_schema(self): args = {'completed_workflow_id': 'ABC123'} state = None tr = Trigger(data=TriggerData('test-trigger', 'workflow_completion', ['ABC123'], args, state=state)) obj_before = tr.to_dict() tr = default_and_validate( tr, trigger_schema(workflow_completion_trigger.params_json_schema)) # state should be defaulted to INACTIVE self.assertNotEqual(obj_before, tr.to_dict())
def _resolve_and_save_trigger(self, entity_id, entity_map, actual_entities_by_node_id, actual_entities_by_unsaved_id): actual_id, unsaved_id = self._resolve(EntityType.trigger, entity_id, entity_map, actual_entities_by_unsaved_id) if actual_id: return actual_id node_id = self._node_id(EntityType.trigger, unsaved_id) trigger = Trigger.from_dict(entity_map['unsaved_entities'][node_id]) assert isinstance(trigger, Trigger) if trigger.data.args and trigger.data.args.get('completed_workflow_id'): trigger.data.args['completed_workflow_id'] = self._resolve_and_save_workflow(trigger.data.args['completed_workflow_id'], entity_map, actual_entities_by_node_id, actual_entities_by_unsaved_id) if trigger.data.args and trigger.data.args.get('event_id'): trigger.data.args['event_id'] = self._resolve_and_save_event(trigger.data.args['event_id'], entity_map, actual_entities_by_node_id, actual_entities_by_unsaved_id) if trigger.data.args and trigger.data.args.get('subscription_id'): trigger.data.args['subscription_id'] = self._resolve_and_save_subscription(trigger.data.args['subscription_id'], entity_map, actual_entities_by_node_id, actual_entities_by_unsaved_id) if trigger.data.workflow_ids: wf_ids = set() for wf_id in trigger.data.workflow_ids: wf_ids.add(self._resolve_and_save_workflow(wf_id, entity_map, actual_entities_by_node_id, actual_entities_by_unsaved_id)) trigger.data.workflow_ids = list(wf_ids) if trigger.data.args.get('completed_trigger_ids'): t_ids = set() for t_id in trigger.data.args['completed_trigger_ids']: t_ids.add(self._resolve_and_save_trigger(t_id, entity_map, actual_entities_by_node_id, actual_entities_by_unsaved_id)) trigger.data.args['completed_trigger_ids'] = list(t_ids) trigger = self._trigger_service.save_trigger(trigger, commit_and_initialize=False, flush=True) actual_entities_by_node_id[node_id] = trigger actual_entities_by_unsaved_id[unsaved_id] = trigger return trigger.id
def post_trigger(): trigger = Trigger.from_dict(request.get_json()) return { 'results': trigger_service().save_trigger(trigger=trigger, user_id=current_user.email).to_dict() }
def _resolve_and_save_trigger(self, entity_id, entity_map, actual_entities_by_node_id, actual_entities_by_unsaved_id): actual_id, unsaved_id = self._resolve(EntityType.trigger, entity_id, entity_map, actual_entities_by_unsaved_id) if actual_id: return actual_id node_id = self._node_id(EntityType.trigger, unsaved_id) trigger = Trigger.from_dict(entity_map['unsaved_entities'][node_id]) assert isinstance(trigger, Trigger) if trigger.data.args and trigger.data.args.get('completed_workflow_id'): trigger.data.args['completed_workflow_id'] = self._resolve_and_save_workflow(trigger.data.args['completed_workflow_id'], entity_map, actual_entities_by_node_id, actual_entities_by_unsaved_id) if trigger.data.args and trigger.data.args.get('event_id'): trigger.data.args['event_id'] = self._resolve_and_save_event(trigger.data.args['event_id'], entity_map, actual_entities_by_node_id, actual_entities_by_unsaved_id) if trigger.data.args and trigger.data.args.get('subscription_id'): trigger.data.args['subscription_id'] = self._resolve_and_save_subscription(trigger.data.args['subscription_id'], entity_map, actual_entities_by_node_id, actual_entities_by_unsaved_id) if trigger.data.workflow_ids: wf_ids = set() for wf_id in trigger.data.workflow_ids: wf_ids.add(self._resolve_and_save_workflow(wf_id, entity_map, actual_entities_by_node_id, actual_entities_by_unsaved_id)) trigger.data.workflow_ids = list(wf_ids) if trigger.data.args.get('completed_trigger_ids'): t_ids = set() for t_id in trigger.data.args['completed_trigger_ids']: t_ids.add(self._resolve_and_save_trigger(t_id, entity_map, actual_entities_by_node_id, actual_entities_by_unsaved_id)) trigger.data.args['completed_trigger_ids'] = list(t_ids) trigger = self._trigger_service.save_trigger(trigger, commit_and_initialize=False, flush=True) actual_entities_by_node_id[node_id] = trigger actual_entities_by_unsaved_id[unsaved_id] = trigger return trigger.id
def test_trigger_schema_invalid(self): with self.assertRaises(DartValidationException) as context: name = None args = {'completed_workflow_id': 'ABC123'} tr = Trigger(data=TriggerData(name, 'workflow_completion', ['ABC123'], args)) # should fail because the name is missing default_and_validate( tr, trigger_schema(workflow_completion_trigger.params_json_schema)) self.assertTrue(isinstance(context.exception, DartValidationException))
def test_crud(self): args = {'completed_workflow_id': self.workflow.id} tr = Trigger(data=TriggerData('test-trigger', 'workflow_completion', [self.workflow.id], args)) posted_tr = self.dart.save_trigger(tr) self.assertEqual(posted_tr.data.to_dict(), tr.data.to_dict()) trigger = self.dart.get_trigger(posted_tr.id) self.assertEqual(posted_tr.to_dict(), trigger.to_dict()) self.dart.delete_trigger(trigger.id) try: self.dart.get_trigger(trigger.id) except DartRequestException as e: self.assertEqual(e.response.status_code, 404) return self.fail('trigger should have been missing after delete!')
def test_crud(self): args = {'completed_workflow_id': self.workflow.id} tr = Trigger(data=TriggerData('test-trigger', 'workflow_completion', [self.workflow.id], args)) posted_tr = self.dart.save_trigger(tr) tr.data.user_id = posted_tr.data.user_id # tags should get the wf_id so we remove it for the test sake posted_tr_data = posted_tr.data.to_dict() posted_tr_data['tags'] = [] tr_data = tr.data.to_dict() tr_data['tags'] = [] self.assertEqual(posted_tr_data, tr_data) trigger = self.dart.get_trigger(posted_tr.id) self.assertEqual(posted_tr.to_dict(), trigger.to_dict()) self.dart.delete_trigger(trigger.id) try: self.dart.get_trigger(trigger.id) except DartRequestException as e: self.assertEqual(e.response.status_code, 404) return self.fail('trigger should have been missing after delete!')
def add_no_op_engine_sub_graphs(config): engine_config = config['engines']['no_op_engine'] opts = engine_config['options'] dart = Dart(opts['dart_host'], opts['dart_port'], opts['dart_api_version']) assert isinstance(dart, Dart) _logger.info('saving no_op_engine sub_graphs') engine_id = None for e in dart.get_engines(): if e.data.name == 'no_op_engine': engine_id = e.id if not engine_id: raise subgraph_definitions = [ SubGraphDefinition(data=SubGraphDefinitionData( name='workflow chaining demo', description='demonstrate workflow chaining', engine_name='no_op_engine', related_type=EntityType.datastore, related_is_a=Relationship.PARENT, workflows=[ Workflow(id=Ref.workflow(1), data=WorkflowData( name='no-op-workflow-chaining-wf1', datastore_id=Ref.parent(), engine_name='no_op_engine', state=WorkflowState.ACTIVE, )), Workflow(id=Ref.workflow(2), data=WorkflowData( name='no-op-workflow-chaining-wf2', datastore_id=Ref.parent(), engine_name='no_op_engine', state=WorkflowState.ACTIVE, )), ], actions=[ Action(id=Ref.action(1), data=ActionData( name=NoOpActionTypes.action_that_succeeds.name, engine_name='no_op_engine', action_type_name=NoOpActionTypes. action_that_succeeds.name, workflow_id=Ref.workflow(1), order_idx=1, state=ActionState.TEMPLATE, )), Action(id=Ref.action(2), data=ActionData( name=NoOpActionTypes.action_that_succeeds.name, action_type_name=NoOpActionTypes. action_that_succeeds.name, engine_name='no_op_engine', workflow_id=Ref.workflow(1), order_idx=2, state=ActionState.TEMPLATE, )), Action(id=Ref.action(3), data=ActionData( name=NoOpActionTypes.action_that_succeeds.name, action_type_name=NoOpActionTypes. action_that_succeeds.name, engine_name='no_op_engine', workflow_id=Ref.workflow(1), order_idx=3, state=ActionState.TEMPLATE, )), Action(id=Ref.action(4), data=ActionData( name=NoOpActionTypes.action_that_succeeds.name, action_type_name=NoOpActionTypes. action_that_succeeds.name, engine_name='no_op_engine', workflow_id=Ref.workflow(1), order_idx=4, state=ActionState.TEMPLATE, )), Action(id=Ref.action(5), data=ActionData( name=NoOpActionTypes.action_that_succeeds.name, action_type_name=NoOpActionTypes. action_that_succeeds.name, engine_name='no_op_engine', workflow_id=Ref.workflow(2), order_idx=1, state=ActionState.TEMPLATE, )), Action(id=Ref.action(6), data=ActionData( name=NoOpActionTypes.action_that_succeeds.name, action_type_name=NoOpActionTypes. action_that_succeeds.name, engine_name='no_op_engine', workflow_id=Ref.workflow(2), order_idx=2, state=ActionState.TEMPLATE, )), Action(id=Ref.action(7), data=ActionData( name=NoOpActionTypes.action_that_fails.name, action_type_name=NoOpActionTypes.action_that_fails. name, engine_name='no_op_engine', workflow_id=Ref.workflow(2), order_idx=3, state=ActionState.TEMPLATE, )), ], triggers=[ Trigger(id=Ref.trigger(1), data=TriggerData( name='no-op-trigger-workflow-completion', trigger_type_name=workflow_completion_trigger.name, workflow_ids=[Ref.workflow(2)], state=TriggerState.ACTIVE, args={'completed_workflow_id': Ref.workflow(1)})), ], )) ] for e in subgraph_definitions: s = dart.save_subgraph_definition(e, engine_id) _logger.info('created subgraph_definition: %s' % s.id)
def post_trigger(): trigger = Trigger.from_dict(request.get_json()) return {'results': trigger_service().save_trigger(trigger).to_dict()}
on_failure_email=['*****@*****.**'], on_success_email=['*****@*****.**'], on_started_email=['*****@*****.**'], )), datastore_id=datastore.id) print 'created workflow: %s' % workflow.id wf_actions = dart.save_actions(actions=[ Action(data=ActionData('consume_subscription', 'consume_subscription', state=ActionState.TEMPLATE, args={ 'subscription_id': subscription.id, 'target_file_format': FileFormat.PARQUET, 'target_row_format': RowFormat.NONE, 'target_compression': Compression.SNAPPY, })), ], workflow_id=workflow.id) print 'created workflow action: %s' % wf_actions[0].id trigger = dart.save_trigger( Trigger(data=TriggerData(name='rmn_direct_trigger_DW-3307', trigger_type_name='subscription_batch', workflow_ids=[workflow.id], args={ 'subscription_id': subscription.id, 'unconsumed_data_size_in_bytes': 16000000 }))) print 'created trigger: %s' % trigger.id
on_failure_email=['*****@*****.**'], on_success_email=['*****@*****.**'], on_started_email=['*****@*****.**'], )), datastore_id=datastore.id) print 'created workflow: %s' % workflow.id a2 = dart.save_actions(actions=[ Action(data=ActionData('consume_subscription', 'consume_subscription', state=ActionState.TEMPLATE, args={ 'subscription_id': subscription.id, 'target_file_format': FileFormat.PARQUET, 'target_row_format': RowFormat.NONE, 'target_compression': Compression.SNAPPY, })), ], workflow_id=workflow.id)[0] print 'created workflow action: %s' % a2.id trigger = dart.save_trigger( Trigger(data=TriggerData(name='owen_eu_parquet_trigger_DW-3213_v3', trigger_type_name='subscription_batch', workflow_ids=[workflow.id], args={ 'subscription_id': subscription.id, 'unconsumed_data_size_in_bytes': 16000000 }))) print 'created trigger: %s' % trigger.id
def patch_trigger(trigger): """ :type trigger: dart.model.trigger.Trigger """ p = JsonPatch(request.get_json()) return update_trigger(trigger, Trigger.from_dict(p.apply(trigger.to_dict())))
datastore.id, state=WorkflowState.ACTIVE, on_failure_email=['*****@*****.**'], on_success_email=['*****@*****.**'], on_started_email=['*****@*****.**'], ) ), datastore.id) print 'created workflow: %s' % workflow.id a0, a1 = dart.save_actions([ Action(data=ActionData('start_datastore', 'start_datastore', state=ActionState.TEMPLATE)), Action(data=ActionData('load_dataset', 'load_dataset', state=ActionState.TEMPLATE, args={ 'dataset_id': dataset.id, 's3_path_start_prefix_inclusive': 's3://example-bucket/prd/beacon/native_app/v2/parquet/snappy/createdpartition=2015-06-27', })), ], workflow_id=workflow.id) print 'created action: %s' % a0.id print 'created action: %s' % a1.id event = dart.save_event(Event(data=EventData('beacon_native_app_to_parquet_emr_job_completion', state=EventState.ACTIVE))) print 'created event: %s' % event.id trigger = dart.save_trigger(Trigger(data=TriggerData( 'beacon_native_app_to_parquet_emr_job_completion_trigger', 'event', [workflow.id], {'event_id': event.id})) ) print 'created trigger: %s' % trigger.id
def post_trigger(): trigger = Trigger.from_dict(request.get_json()) return {'results': trigger_service().save_trigger(trigger=trigger, user_id=current_user.email).to_dict()}
def _get_engineless_static_subgraphs_by_related_type(graph_entity_service): sub_graph_map = {} d_entity_models = graph_entity_service.to_entity_models_with_randomized_ids( [ Dataset(id=Ref.dataset(1), data=DatasetData(None, None, None, None, None, columns=[], partitions=[])) ]) e_entity_models = graph_entity_service.to_entity_models_with_randomized_ids( [Event(id=Ref.event(1), data=EventData('event'))]) sub_graph_map[None] = [ SubGraph( name='dataset', description='create a new dataset entity', related_type=None, related_is_a=None, graph=graph_entity_service.to_graph(None, d_entity_models), entity_map=graph_entity_service.to_entity_map(d_entity_models), icon='⬟', ), SubGraph( name='event', description='create a new event entity', related_type=None, related_is_a=None, graph=graph_entity_service.to_graph(None, e_entity_models), entity_map=graph_entity_service.to_entity_map(e_entity_models), icon='★', ), ] entity_models = graph_entity_service.to_entity_models_with_randomized_ids([ Subscription(id=Ref.subscription(1), data=SubscriptionData('subscription', Ref.parent())) ]) sub_graph_map[EntityType.dataset] = [ SubGraph( name='subscription', description='create a new subscription entity', related_type=EntityType.dataset, related_is_a=Relationship.PARENT, graph=graph_entity_service.to_graph(None, entity_models), entity_map=graph_entity_service.to_entity_map(entity_models), icon='⬢', ), ] entity_models = graph_entity_service.to_entity_models_with_randomized_ids([ Trigger(id=Ref.trigger(1), data=TriggerData(name='%s_trigger' % event_trigger.name, trigger_type_name=event_trigger.name, state=TriggerState.INACTIVE, workflow_ids=[], args={'event_id': Ref.parent()})) ]) sub_graph_map[EntityType.event] = [ SubGraph( name='event trigger', description='create a new event trigger entity', related_type=EntityType.event, related_is_a=Relationship.PARENT, graph=graph_entity_service.to_graph(None, entity_models), entity_map=graph_entity_service.to_entity_map(entity_models), icon='▼', ), ] entity_models = graph_entity_service.to_entity_models_with_randomized_ids([ Trigger(id=Ref.trigger(1), data=TriggerData( name='%s_trigger' % subscription_batch_trigger.name, trigger_type_name=subscription_batch_trigger.name, state=TriggerState.INACTIVE, workflow_ids=[], args={ 'subscription_id': Ref.parent(), 'unconsumed_data_size_in_bytes': 1000000 })) ]) sub_graph_map[EntityType.subscription] = [ SubGraph( name='subscription batch trigger', description='create a new subscription batch trigger entity', related_type=EntityType.subscription, related_is_a=Relationship.PARENT, graph=graph_entity_service.to_graph(None, entity_models), entity_map=graph_entity_service.to_entity_map(entity_models), icon='▼', ), ] return sub_graph_map
)), datastore_id=datastore.id) print 'created workflow: %s' % workflow.id a2 = dart.save_actions(actions=[ Action(data=ActionData('consume_subscription', 'consume_subscription', state=ActionState.TEMPLATE, args={ 'subscription_id': subscription.id, 'target_file_format': FileFormat.PARQUET, 'target_row_format': RowFormat.NONE, 'target_compression': Compression.SNAPPY, })), ], workflow_id=workflow.id)[0] print 'created workflow action: %s' % a2.id trigger = dart.save_trigger( Trigger(data=TriggerData( name='owen_eu_parquet_trigger_DW-3411_v1', trigger_type_name='subscription_batch', workflow_ids=[workflow.id], args={ 'subscription_id': subscription.id, 'unconsumed_data_size_in_bytes': 16000000 }, state=TriggerState.ACTIVE, ))) print 'created trigger: %s' % trigger.id
def post_trigger(): trigger = Trigger.from_dict(request.get_json()) return {'results': trigger_service().save_trigger(trigger).to_dict()}
def put_trigger(trigger): """ :type trigger: dart.model.trigger.Trigger """ updated_trigger = Trigger.from_dict(request.get_json()) trigger = trigger_service().update_trigger_state(trigger, updated_trigger.data.state) return {'results': trigger.to_dict()}
def put_trigger(trigger): """ :type trigger: dart.model.trigger.Trigger """ updated_trigger = Trigger.from_dict(request.get_json()) trigger = trigger_service().update_trigger_state( trigger, updated_trigger.data.state) return {'results': trigger.to_dict()}
def setUp(self): dart = Dart(host='localhost', port=5000) """ :type dart: dart.client.python.dart_client.Dart """ self.dart = dart cs = [Column('c1', DataType.VARCHAR, 50), Column('c2', DataType.BIGINT)] df = DataFormat(FileFormat.PARQUET, RowFormat.NONE) dataset_data = DatasetData('test-dataset0', 'test_dataset_table0', 's3://test/dataset/0/%s' + random_id(), df, cs) self.dataset0 = self.dart.save_dataset(Dataset(data=dataset_data)) cs = [Column('c1', DataType.VARCHAR, 50), Column('c2', DataType.BIGINT)] df = DataFormat(FileFormat.PARQUET, RowFormat.NONE) dataset1_location = 's3://test/dataset/1/%s' + random_id() dataset_data = DatasetData('test-dataset1', 'test_dataset_table1', dataset1_location, df, cs) self.dataset1 = self.dart.save_dataset(Dataset(data=dataset_data)) cs = [Column('c1', DataType.VARCHAR, 50), Column('c2', DataType.BIGINT)] df = DataFormat(FileFormat.PARQUET, RowFormat.NONE) dataset_data = DatasetData('test-dataset2-no-show', 'test_dataset_table2', 's3://test/dataset/2/%s' + random_id(), df, cs) self.dataset2 = self.dart.save_dataset(Dataset(data=dataset_data)) s = Subscription(data=SubscriptionData('test-subscription0', self.dataset0.id)) self.subscription0 = self.dart.save_subscription(s) s = Subscription(data=SubscriptionData('test-subscription2-no-show', self.dataset2.id)) self.subscription2 = self.dart.save_subscription(s) dst_args = {'action_sleep_time_in_seconds': 0} dst = Datastore(data=DatastoreData('test-datastore0', 'no_op_engine', args=dst_args, state=DatastoreState.TEMPLATE)) self.datastore0 = self.dart.save_datastore(dst) dst = Datastore(data=DatastoreData('test-datastore1', 'no_op_engine', args=dst_args, state=DatastoreState.TEMPLATE)) self.datastore1 = self.dart.save_datastore(dst) dst = Datastore(data=DatastoreData('test-datastore2-no-show', 'no_op_engine', args=dst_args, state=DatastoreState.ACTIVE)) self.datastore2 = self.dart.save_datastore(dst) wf0 = Workflow(data=WorkflowData('test-workflow0', self.datastore0.id, state=WorkflowState.ACTIVE)) self.workflow0 = self.dart.save_workflow(wf0, self.datastore0.id) wf1 = Workflow(data=WorkflowData('test-workflow1', self.datastore1.id, state=WorkflowState.ACTIVE)) self.workflow1 = self.dart.save_workflow(wf1, self.datastore1.id) wf2 = Workflow(data=WorkflowData('test-workflow2-no-show', self.datastore2.id, state=WorkflowState.ACTIVE)) self.workflow2 = self.dart.save_workflow(wf2, self.datastore2.id) a_args = {'source_hdfs_path': 'hdfs:///user/hive/warehouse/test', 'destination_s3_path': dataset1_location} a00 = Action(data=ActionData(NoOpActionTypes.action_that_succeeds.name, NoOpActionTypes.action_that_succeeds.name, state=ActionState.TEMPLATE)) a01 = Action(data=ActionData(NoOpActionTypes.consume_subscription.name, NoOpActionTypes.consume_subscription.name, {'subscription_id': self.subscription0.id}, state=ActionState.TEMPLATE)) a02 = Action(data=ActionData(NoOpActionTypes.action_that_succeeds.name, NoOpActionTypes.action_that_succeeds.name, state=ActionState.TEMPLATE)) a03 = Action(data=ActionData(NoOpActionTypes.copy_hdfs_to_s3_action.name, NoOpActionTypes.copy_hdfs_to_s3_action.name, a_args, state=ActionState.TEMPLATE)) a04 = Action(data=ActionData(NoOpActionTypes.action_that_succeeds.name, NoOpActionTypes.action_that_succeeds.name, state=ActionState.TEMPLATE)) self.action00, self.action01, self.action02, self.action03, self.action04 = \ self.dart.save_actions([a00, a01, a02, a03, a04], workflow_id=self.workflow0.id) a10 = Action(data=ActionData(NoOpActionTypes.load_dataset.name, NoOpActionTypes.load_dataset.name, {'dataset_id': self.dataset1.id}, state=ActionState.TEMPLATE)) self.action10 = self.dart.save_actions([a10], workflow_id=self.workflow1.id) a20 = Action(data=ActionData(NoOpActionTypes.action_that_succeeds.name, NoOpActionTypes.action_that_succeeds.name, state=ActionState.HAS_NEVER_RUN)) a21 = Action(data=ActionData(NoOpActionTypes.load_dataset.name, NoOpActionTypes.load_dataset.name, {'dataset_id': self.dataset2.id}, state=ActionState.TEMPLATE)) self.action20 = self.dart.save_actions([a20], datastore_id=self.datastore2.id) self.action21 = self.dart.save_actions([a21], workflow_id=self.workflow2.id) self.event1 = self.dart.save_event(Event(data=EventData('test-event1', state=EventState.ACTIVE))) self.event2 = self.dart.save_event(Event(data=EventData('test-event2-no-show', state=EventState.ACTIVE))) tr_args = {'event_id': self.event1.id} tr = Trigger(data=TriggerData('test-event-trigger1', 'event', [self.workflow1.id], tr_args, TriggerState.ACTIVE)) self.event_trigger1 = self.dart.save_trigger(tr) tr_args = {'event_id': self.event2.id} tr = Trigger(data=TriggerData('test-event-trigger2-no-show', 'event', [self.workflow2.id], tr_args, TriggerState.ACTIVE)) self.event_trigger2 = self.dart.save_trigger(tr) st_args = {'fire_after': 'ALL', 'completed_trigger_ids': [self.event_trigger1.id]} st = Trigger(data=TriggerData('test-super-trigger1', 'super', None, st_args, TriggerState.ACTIVE)) self.super_trigger1 = self.dart.save_trigger(st) st_args = {'fire_after': 'ANY', 'completed_trigger_ids': [self.super_trigger1.id]} st = Trigger(data=TriggerData('test-super-trigger2', 'super', [self.workflow1.id], st_args, TriggerState.ACTIVE)) self.super_trigger2 = self.dart.save_trigger(st)
def put_trigger(trigger): """ :type trigger: dart.model.trigger.Trigger """ return update_trigger(trigger, Trigger.from_dict(request.get_json()))
def put_trigger(trigger): """ :type trigger: dart.model.trigger.Trigger """ return update_trigger(trigger, Trigger.from_dict(request.get_json()))
def patch_trigger(trigger): """ :type trigger: dart.model.trigger.Trigger """ p = JsonPatch(request.get_json()) return update_trigger(trigger, Trigger.from_dict(p.apply(trigger.to_dict())))
def _get_static_subgraphs_by_related_type(engine, graph_entity_service): engine_name = engine.data.name sub_graph_map = {EntityType.workflow: []} for action_type in engine.data.supported_action_types: entity_models = graph_entity_service.to_entity_models_with_randomized_ids( [ Action( id=Ref.action(1), data=ActionData( name=action_type.name, action_type_name=action_type.name, engine_name=engine_name, workflow_id=Ref.parent(), state=ActionState.TEMPLATE, args={} if action_type.params_json_schema else None)) ]) sub_graph_map[EntityType.workflow].append( SubGraph( name=action_type.name, description=action_type.description, related_type=EntityType.workflow, related_is_a=Relationship.PARENT, graph=graph_entity_service.to_graph(None, entity_models), entity_map=graph_entity_service.to_entity_map(entity_models), icon='●', )) entity_models = graph_entity_service.to_entity_models_with_randomized_ids([ Trigger(id=Ref.trigger(1), data=TriggerData( name='%s_trigger' % workflow_completion_trigger.name, trigger_type_name=workflow_completion_trigger.name, state=TriggerState.INACTIVE, workflow_ids=[], args={'completed_workflow_id': Ref.parent()})) ]) sub_graph_map[EntityType.workflow].extend([ SubGraph( name='workflow completion trigger', description='create a new workflow_completion trigger entity', related_type=EntityType.workflow, related_is_a=Relationship.PARENT, graph=graph_entity_service.to_graph(None, entity_models), entity_map=graph_entity_service.to_entity_map(entity_models), icon='▼', ), ]) entity_models = graph_entity_service.to_entity_models_with_randomized_ids([ Trigger(id=Ref.trigger(1), data=TriggerData( name='%s_trigger' % scheduled_trigger.name, trigger_type_name=scheduled_trigger.name, state=TriggerState.INACTIVE, workflow_ids=[Ref.child()], )) ]) sub_graph_map[EntityType.workflow].extend([ SubGraph( name='scheduled trigger', description='create a new scheduled trigger entity', related_type=EntityType.workflow, related_is_a=Relationship.CHILD, graph=graph_entity_service.to_graph(None, entity_models), entity_map=graph_entity_service.to_entity_map(entity_models), icon='▼', ), ]) entity_models = graph_entity_service.to_entity_models_with_randomized_ids([ Trigger(id=Ref.trigger(1), data=TriggerData( name='%s_trigger' % super_trigger.name, trigger_type_name=super_trigger.name, state=TriggerState.INACTIVE, workflow_ids=[Ref.child()], )) ]) sub_graph_map[EntityType.workflow].extend([ SubGraph( name='super trigger', description='create a new super trigger entity', related_type=EntityType.workflow, related_is_a=Relationship.CHILD, graph=graph_entity_service.to_graph(None, entity_models), entity_map=graph_entity_service.to_entity_map(entity_models), icon='▼', ), ]) entity_models = graph_entity_service.to_entity_models_with_randomized_ids([ Workflow(id=Ref.workflow(1), data=WorkflowData(name='workflow', datastore_id=Ref.parent(), engine_name=engine_name, state=WorkflowState.INACTIVE)) ]) sub_graph_map[EntityType.datastore] = [ SubGraph( name='workflow', description='create a new workflow entity', related_type=EntityType.datastore, related_is_a=Relationship.PARENT, graph=graph_entity_service.to_graph(None, entity_models), entity_map=graph_entity_service.to_entity_map(entity_models), icon='◆', ) ] for action_type in engine.data.supported_action_types: entity_models = graph_entity_service.to_entity_models_with_randomized_ids( [ Action( id=Ref.action(1), data=ActionData( name=action_type.name, action_type_name=action_type.name, engine_name=engine_name, datastore_id=Ref.parent(), state=ActionState.HAS_NEVER_RUN, args={} if action_type.params_json_schema else None)) ]) sub_graph_map[EntityType.datastore].append( SubGraph( name=action_type.name, description=action_type.description, related_type=EntityType.datastore, related_is_a=Relationship.PARENT, graph=graph_entity_service.to_graph(None, entity_models), entity_map=graph_entity_service.to_entity_map(entity_models), icon='●', )) return sub_graph_map