def conditional_insert_subscription_element(subscription, s3_path, size): # SQLAlchemy does not support conditional inserts as a part of its expression language. Furthermore, # this form of conditional update is required until postgres 9.5 is out and supported by RDS: # # http://www.postgresql.org/docs/devel/static/sql-insert.html#SQL-ON-CONFLICT # sql = """ INSERT INTO subscription_element ( id, version_id, created, updated, subscription_id, s3_path, file_size, state ) SELECT :id, 0, NOW(), NOW(), :sid, :s3_path, :size, :state WHERE NOT EXISTS (SELECT NULL FROM subscription_element WHERE subscription_id = :sid AND s3_path = :s3_path) """ sid = subscription.id state = SubscriptionElementState.UNCONSUMED statement = text(sql).bindparams(id=random_id(), sid=sid, s3_path=s3_path, size=size, state=state) results = db.session.execute(statement) if results.rowcount != 1: db.session.rollback() return False else: db.session.commit() return True
def save_datastore(self, datastore, commit_and_handle_state_change=True, flush=False): """ :type datastore: dart.model.datastore.Datastore """ schema = self.get_schema(datastore) datastore = self.default_and_validate_datastore(datastore, schema) datastore.id = random_id() secrets = {} datastore_dict = datastore.to_dict() purge_secrets(datastore_dict, schema, secrets) datastore = Datastore.from_dict(datastore_dict) for k, v in secrets.iteritems(): self._secrets.put('dart-datastore-%s-%s' % (datastore.id, k), v) self._set_s3_paths(datastore) datastore_dao = DatastoreDao() datastore_dao.id = datastore.id datastore_dao.data = datastore.data.to_dict() db.session.add(datastore_dao) if flush: db.session.flush() datastore = datastore_dao.to_model() if commit_and_handle_state_change: db.session.commit() datastore = datastore_dao.to_model() self.handle_datastore_state_change(datastore, None, datastore_dao.data['state']) return datastore
def save_datastore(self, datastore, commit_and_handle_state_change=True, flush=False): """ :type datastore: dart.model.datastore.Datastore """ schema = self.get_schema(datastore) datastore = self.default_and_validate_datastore(datastore, schema) datastore.id = random_id() secrets = {} datastore_dict = datastore.to_dict() purge_secrets(datastore_dict, schema, secrets) datastore = Datastore.from_dict(datastore_dict) for k, v in secrets.iteritems(): self._secrets.put('dart-datastore-%s-%s' % (datastore.id, k), v) self._set_s3_paths(datastore) datastore_dao = DatastoreDao() datastore_dao.id = datastore.id datastore_dao.data = datastore.data.to_dict() db.session.add(datastore_dao) if flush: db.session.flush() datastore = datastore_dao.to_model() if commit_and_handle_state_change: db.session.commit() datastore = datastore_dao.to_model() self.handle_datastore_state_change(datastore, None, datastore_dao.data['state']) return datastore
def save_trigger(self, trigger, commit_and_initialize=True, flush=False): """ :type trigger: dart.model.trigger.Trigger """ trigger_type_name = trigger.data.trigger_type_name if trigger_type_name == self._manual_trigger_processor.trigger_type().name: raise DartValidationException('manual triggers cannot be saved') trigger_processor = self._trigger_processors.get(trigger_type_name) if not trigger_processor: raise DartValidationException('unknown trigger_type_name: %s' % trigger_type_name) assert isinstance(trigger_processor, TriggerProcessor) trigger = default_and_validate(trigger, trigger_schema(trigger_processor.trigger_type().params_json_schema)) trigger_dao = TriggerDao() trigger_dao.id = random_id() trigger_dao.data = trigger.data.to_dict() db.session.add(trigger_dao) if flush: db.session.flush() trigger = trigger_dao.to_model() if commit_and_initialize: db.session.commit() trigger = trigger_dao.to_model() try: trigger_processor.initialize_trigger(trigger, self) except: db.session.delete(trigger_dao) db.session.commit() raise return trigger
def save_trigger(self, trigger, commit_and_initialize=True, flush=False): """ :type trigger: dart.model.trigger.Trigger """ trigger_type_name = trigger.data.trigger_type_name if trigger_type_name == self._manual_trigger_processor.trigger_type( ).name: raise DartValidationException('manual triggers cannot be saved') trigger_processor = self._trigger_processors.get(trigger_type_name) if not trigger_processor: raise DartValidationException('unknown trigger_type_name: %s' % trigger_type_name) assert isinstance(trigger_processor, TriggerProcessor) trigger = default_and_validate( trigger, trigger_schema( trigger_processor.trigger_type().params_json_schema)) trigger_dao = TriggerDao() trigger_dao.id = random_id() trigger_dao.data = trigger.data.to_dict() db.session.add(trigger_dao) if flush: db.session.flush() trigger = trigger_dao.to_model() if commit_and_initialize: db.session.commit() trigger = trigger_dao.to_model() try: trigger_processor.initialize_trigger(trigger, self) except: db.session.delete(trigger_dao) db.session.commit() raise return trigger
def save_accounting_event(accounting_event, commit=True, flush=False): """ :type accounting_event: dart.model.accounting.Accounting """ accounting_dao = AccountingDao() accounting_dao.id = random_id() accounting_dao.user_id = accounting_event.user_id accounting_dao.state = accounting_event.state accounting_dao.entity = accounting_event.entity accounting_dao.params = accounting_event.params accounting_dao.return_code = accounting_event.return_code accounting_dao.api_version = accounting_event.api_version accounting_dao.extra = accounting_event.extra db.session.add(accounting_dao) try: if flush: db.session.flush() if commit: db.session.commit() # accounting_event = accounting_dao.to_model() return accounting_event except SqlAlchemyIntegrityError as e: if hasattr(e, 'orig') and isinstance(e.orig, PostgresIntegrityError) and e.orig.pgcode == '23505': raise DartValidationException('name already exists: %s' % dataset.data.name) raise e
def save_actions(self, actions, engine_name, datastore=None, commit=True, flush=False): """ :type actions: list[dart.model.action.Action] :type datastore: dart.model.datastore.Datastore """ engine = self._engine_service.get_engine_by_name(engine_name) assert isinstance(engine, Engine) action_types_by_name = {at.name: at for at in engine.data.supported_action_types} action_daos = [] max_order_idx = ActionService._get_max_order_idx(datastore.id) + 1 if datastore else 0 for action in actions: action.data.engine_name = engine_name if not action.data.order_idx: action.data.order_idx = max_order_idx max_order_idx = action.data.order_idx + 1 action_dao = ActionDao() action_dao.id = random_id() action_type = action_types_by_name.get(action.data.action_type_name) action = self.default_and_validate_action(action, action_type) action_dao.data = action.data.to_dict() db.session.add(action_dao) action_daos.append(action_dao) if flush: db.session.flush() if commit: db.session.commit() return [a.to_model() for a in action_daos]
def save_accounting_event(accounting_event, commit=True, flush=False): """ :type accounting_event: dart.model.accounting.Accounting """ accounting_dao = AccountingDao() accounting_dao.id = random_id() accounting_dao.user_id = accounting_event.user_id accounting_dao.state = accounting_event.state accounting_dao.entity = accounting_event.entity accounting_dao.params = accounting_event.params accounting_dao.return_code = accounting_event.return_code accounting_dao.api_version = accounting_event.api_version accounting_dao.extra = accounting_event.extra db.session.add(accounting_dao) try: if flush: db.session.flush() if commit: db.session.commit() # accounting_event = accounting_dao.to_model() return accounting_event except SqlAlchemyIntegrityError as e: if hasattr(e, 'orig') and isinstance( e.orig, PostgresIntegrityError) and e.orig.pgcode == '23505': raise DartValidationException('name already exists: %s' % dataset.data.name) raise e
def save_workflow_instance(workflow, trigger_type, trigger_id, state, log_info=None): """ :type workflow: dart.model.workflow.Workflow :type trigger_type: dart.model.trigger.TriggerType """ wf_instance_dao = WorkflowInstanceDao() wf_instance_dao.id = random_id() wf_data = workflow.data wf_data_tags = wf_data.tags if(wf_data.tags) else [] if (log_info and log_info.get('wf_uuid')): wf_data_tags.append(log_info.get('wf_uuid')) user_id = 'anonymous' if (log_info and log_info.get('user_id')): user_id = log_info.get('user_id') data = WorkflowInstanceData( workflow_id=workflow.id, engine_name=wf_data.engine_name, state=state, trigger_type=trigger_type.name, trigger_id=trigger_id, queued_time=datetime.now(), tags=wf_data_tags, user_id=user_id, ) wf_instance_dao.data = data.to_dict() db.session.add(wf_instance_dao) db.session.commit() return wf_instance_dao.to_model()
def save_workflow_instance(workflow, trigger_type, trigger_id, state, log_info=None, retry_num=0): """ :type workflow: dart.model.workflow.Workflow :type trigger_type: dart.model.trigger.TriggerType """ wf_instance_dao = WorkflowInstanceDao() wf_instance_dao.id = random_id() wf_data = workflow.data wf_data_tags = wf_data.tags if (wf_data.tags) else [] if (log_info and log_info.get('wf_uuid')): wf_data_tags.append(log_info.get('wf_uuid')) user_id = 'anonymous' if (log_info and log_info.get('user_id')): user_id = log_info.get('user_id') data = WorkflowInstanceData( workflow_id=workflow.id, engine_name=wf_data.engine_name, state=state, trigger_type=trigger_type.name, trigger_id=trigger_id, queued_time=datetime.now(), tags=wf_data_tags, user_id=user_id, retry_num=retry_num, ) wf_instance_dao.data = data.to_dict() db.session.add(wf_instance_dao) db.session.commit() return wf_instance_dao.to_model()
def create_all(): db.create_all() for mutex in Mutexes.all(): sql = """ INSERT INTO mutex (id, version_id, created, updated, name, state) SELECT :id, 0, NOW(), NOW(), :name, :state WHERE NOT EXISTS (SELECT NULL FROM mutex WHERE name = :name) """ statement = text(sql).bindparams(id=random_id(), name=mutex, state=MutexState.READY) db.session.execute(statement) db.session.commit() # config values for dart_client_key/secret are extracted. if AUTH_CONFIG.get('use_auth'): if AUTH_CONFIG.get('dart_client_key') and AUTH_CONFIG.get('dart_client_secret'): _credential = AUTH_CONFIG.get('dart_client_key') _secret = AUTH_CONFIG.get('dart_client_secret') else: raise Exception('dart_client_key and dart_client_secret must both exist.') else: # The credential/secret default values are set in order to prevent exception while calculating to hmac. _credential = 'cred' _secret = 'secret' # We set a user for dart_client with a apikey/secret (read from config) so that dart_client can work. # api auth expects a user to exist in the user table and have an entry in the api_key table (with key/secret values set). populate_dart_client_user(DART_CLIENT_NAME) populate_dart_client_apikeys(_credential, _secret, DART_CLIENT_NAME) # populate user and keys for external service (not dart client) populate_user_api_secret_keys() return 'OK'
def save_actions(self, actions, engine_name, datastore=None, commit=True, flush=False): """ :type actions: list[dart.model.action.Action] :type datastore: dart.model.datastore.Datastore """ engine = self._engine_service.get_engine_by_name(engine_name) assert isinstance(engine, Engine) action_types_by_name = {at.name: at for at in engine.data.supported_action_types} action_daos = [] max_order_idx = ActionService._get_max_order_idx(datastore.id) + 1 if datastore else 0 for action in actions: action.data.engine_name = engine_name if not action.data.order_idx: action.data.order_idx = max_order_idx max_order_idx = action.data.order_idx + 1 action_dao = ActionDao() action_dao.id = random_id() action_type = action_types_by_name.get(action.data.action_type_name) action = self.default_and_validate_action(action, action_type) action_dao.data = action.data.to_dict() db.session.add(action_dao) action_daos.append(action_dao) if flush: db.session.flush() if commit: db.session.commit() return [a.to_model() for a in action_daos]
def conditional_insert_subscription_element(subscription, s3_path, size): # SQLAlchemy does not support conditional inserts as a part of its expression language. Furthermore, # this form of conditional update is required until postgres 9.5 is out and supported by RDS: # # http://www.postgresql.org/docs/devel/static/sql-insert.html#SQL-ON-CONFLICT # sql = """ INSERT INTO subscription_element ( id, version_id, created, updated, subscription_id, s3_path, file_size, state ) SELECT :id, 0, NOW(), NOW(), :sid, :s3_path, :size, :state WHERE NOT EXISTS (SELECT NULL FROM subscription_element WHERE subscription_id = :sid AND s3_path = :s3_path) """ sid = subscription.id state = SubscriptionElementState.UNCONSUMED statement = text(sql).bindparams(id=random_id(), sid=sid, s3_path=s3_path, size=size, state=state) results = db.session.execute(statement) if results.rowcount != 1: db.session.rollback() return False else: db.session.commit() return True
def generate_subscription_elements(self, subscription): """ :type subscription: dart.model.subscription.Subscription """ _update_subscription_state(subscription, SubscriptionState.GENERATING) dataset = self._dataset_service.get_dataset( subscription.data.dataset_id) conn = boto.connect_s3() bucket = get_bucket(conn, dataset.data.location) s3_keys = yield_s3_keys( bucket, dataset.data.location, subscription.data.s3_path_start_prefix_inclusive, subscription.data.s3_path_end_prefix_exclusive, subscription.data.s3_path_regex_filter, ) elements = [] subscription_element_dict = {} for i, key_obj in enumerate(s3_keys): sid = subscription.id s3_path = get_s3_path(key_obj) state = SubscriptionElementState.UNCONSUMED now = datetime.now() subscription_element_dict = { 'id': random_id(), 'version_id': 0, 'created': now, 'updated': now, 'subscription_id': sid, 's3_path': s3_path, 'file_size': key_obj.size, 'state': state } elements.append(subscription_element_dict) batch_size_reached = (i + 1) % _batch_size == 0 if batch_size_reached: self._insert_elements(elements) elements = [] if len(elements) > 0: self._insert_elements(elements) _update_subscription_state(subscription, SubscriptionState.ACTIVE) # Now that the subscription is ACTIVE, s3 events for new files will cause conditional inserts to be # performed to keep the subscription up to date. However, in the time it took for the subscription # elements to be generated, s3 events for new objects could have been missed. So we will do one final # s3 list operation (starting with the last inserted key) to fill in the potential gap. s3_keys = yield_s3_keys( bucket, dataset.data.location, subscription_element_dict.get('s3_path'), subscription.data.s3_path_end_prefix_exclusive, subscription.data.s3_path_regex_filter, ) for key_obj in s3_keys: self.conditional_insert_subscription_element( subscription, get_s3_path(key_obj), key_obj.size)
def _handle_docker_concerns(self, cwl_image, eng_cfg, misc_log_group_name, output_config, syslog_log_group_name): if 'docker' in self.stacks_to_skip: _logger.info('skipping docker concerns') return _logger.info( 'configuring and building cloudwatch logs docker image (a special snowflake)' ) dart_root = dart_root_relative_path() r_id = random_id() values = (dart_root, r_id) call( 'cd %s && cd .. && git clone https://github.com/awslabs/ecs-cloudwatch-logs dart-cwl-%s' % values) docker_init = dart_root_relative_path('tools', 'docker', 'docker-local-init.sh') with open(dart_root_relative_path('aws', 'cloudwatch-logs', 'awslogs_template.conf')) as cwl_conf_template, \ open(dart_root_relative_path('..', 'dart-cwl-%s/awslogs.conf' % r_id), mode='w') as cwl_conf: contents = cwl_conf_template.read() contents = contents.replace('{DART_LOG_GROUP_SYSLOG}', syslog_log_group_name) contents = contents.replace('{DART_LOG_GROUP_MISC}', misc_log_group_name) cwl_conf.write(contents) cwl_root = dart_root_relative_path('..', 'dart-cwl-%s' % r_id) call('source %s && cd %s && docker build -f Dockerfile -t %s .' % (docker_init, cwl_root, cwl_image)) _logger.info('running grunt build') call('cd %s && grunt build' % dart_root_relative_path('src', 'python', 'dart', 'web', 'ui')) _logger.info('building other docker images') for repo_name in [ rn for rn in output_config['ecr']['repo_names'] if not rn.endswith('cloudwatchlogs') ]: version = eng_cfg['emr_engine']['options'][ 'impala_version'] if 'impala' in repo_name else '1.0.0' docker_img = self._docker_image(repo_name, output_config, version=version) docker_file_suffix = repo_name.split('/')[-1] values = (docker_init, dart_root, docker_file_suffix, docker_img) call( 'source %s && cd %s && docker build -f tools/docker/Dockerfile-%s -t %s .' % values) _logger.info('pushing docker images') cmd = ('source %s && cd %s && $(aws ecr get-login)' % (docker_init, dart_root)) + ' && docker push %s' for repo_name in output_config['ecr']['repo_names']: version = eng_cfg['emr_engine']['options'][ 'impala_version'] if 'impala' in repo_name else '1.0.0' call(cmd % self._docker_image(repo_name, output_config, version=version))
def reserve_subscription_elements(element_ids): # because this is called by the trigger worker (always a single consumer), # we shouldn't have to deal with optimistic locking db.session.execute( update(SubscriptionElementDao).where( SubscriptionElementDao.id.in_(element_ids)).values( state=SubscriptionElementState.RESERVED, batch_id=random_id())) db.session.commit()
def generate_subscription_elements(self, subscription): """ :type subscription: dart.model.subscription.Subscription """ _update_subscription_state(subscription, SubscriptionState.GENERATING) dataset = self._dataset_service.get_dataset(subscription.data.dataset_id) conn = boto.connect_s3() bucket = get_bucket(conn, dataset.data.location) s3_keys = yield_s3_keys( bucket, dataset.data.location, subscription.data.s3_path_start_prefix_inclusive, subscription.data.s3_path_end_prefix_exclusive, subscription.data.s3_path_regex_filter, ) elements = [] subscription_element_dict = {} for i, key_obj in enumerate(s3_keys): sid = subscription.id s3_path = get_s3_path(key_obj) state = SubscriptionElementState.UNCONSUMED now = datetime.now() subscription_element_dict = { 'id': random_id(), 'version_id': 0, 'created': now, 'updated': now, 'subscription_id': sid, 's3_path': s3_path, 'file_size': key_obj.size, 'state': state } elements.append(subscription_element_dict) batch_size_reached = (i + 1) % _batch_size == 0 if batch_size_reached: self._insert_elements(elements) elements = [] if len(elements) > 0: self._insert_elements(elements) _update_subscription_state(subscription, SubscriptionState.ACTIVE) # Now that the subscription is ACTIVE, s3 events for new files will cause conditional inserts to be # performed to keep the subscription up to date. However, in the time it took for the subscription # elements to be generated, s3 events for new objects could have been missed. So we will do one final # s3 list operation (starting with the last inserted key) to fill in the potential gap. s3_keys = yield_s3_keys( bucket, dataset.data.location, subscription_element_dict.get('s3_path'), subscription.data.s3_path_end_prefix_exclusive, subscription.data.s3_path_regex_filter, ) for key_obj in s3_keys: self.conditional_insert_subscription_element(subscription, get_s3_path(key_obj), key_obj.size)
def save_trigger(self, trigger, commit_and_initialize=True, flush=False, user_id=None): wf_uuid = uuid.uuid4().hex # to avoid uuid serialization issues trigger.data.tags = trigger.data.tags if (trigger.data.tags) else [] trigger.data.tags.append(wf_uuid) if user_id: trigger.data.user_id = user_id """ :type trigger: dart.model.trigger.Trigger """ trigger_type_name = trigger.data.trigger_type_name if trigger_type_name == self._manual_trigger_processor.trigger_type( ).name: raise DartValidationException('manual triggers cannot be saved') if trigger_type_name == self._retry_trigger_processor.trigger_type( ).name: raise DartValidationException('retry triggers cannot be saved') trigger_processor = self._trigger_processors.get(trigger_type_name) if not trigger_processor: raise DartValidationException('unknown trigger_type_name: %s' % trigger_type_name) assert isinstance(trigger_processor, TriggerProcessor) trigger = default_and_validate( trigger, trigger_schema( trigger_processor.trigger_type().params_json_schema)) trigger_dao = TriggerDao() trigger_dao.id = random_id() if trigger_type_name == 'subscription_batch': sub = self._subscription_service.get_subscription( trigger.data.args['subscription_id']) if sub.data.nudge_id: response = self.update_nudge_with_trigger( sub.data.nudge_id, trigger.data.args['unconsumed_data_size_in_bytes'], trigger_dao.id, trigger.data.trigger_type_name) assert (response.status_code == 200) trigger_dao.data = trigger.data.to_dict() db.session.add(trigger_dao) if flush: db.session.flush() trigger = trigger_dao.to_model() if commit_and_initialize: db.session.commit() trigger = trigger_dao.to_model() try: trigger_processor.initialize_trigger(trigger, self) except: db.session.delete(trigger_dao) db.session.commit() raise return trigger
def reserve_subscription_elements(element_ids): # because this is called by the trigger worker (always a single consumer), # we shouldn't have to deal with optimistic locking db.session.execute( update(SubscriptionElementDao) .where(SubscriptionElementDao.id.in_(element_ids)) .values( state=SubscriptionElementState.RESERVED, batch_id=random_id() ) ) db.session.commit()
def save_api_key(api_key, commit=True, flush=False): """ :type api_key: dart.model.api_key.ApiKey """ key_dao = ApiKeyDao() for key, value in user.to_dict().iteritems(): setattr(key_dao, key, value) if not key_dao.id: key_dao.id = random_id() db.session.merge(key_dao) if flush: db.session.flush() if commit: db.session.commit() return key_dao.to_model()
def save_user(user, commit=True, flush=False): """ :type user: dart.model.user.User """ user_dao = UserDao() for key, value in user.to_dict().iteritems(): setattr(user_dao, key, value) if not user_dao.id: user_dao.id = random_id() db.session.merge(user_dao) if flush: db.session.flush() if commit: db.session.commit() return user_dao.to_model()
def save_user(user, commit=True, flush=False): """ :type user: dart.model.user.User """ user_dao = UserDao() for key, value in user.to_dict().iteritems(): setattr(user_dao, key, value) if not user_dao.id: user_dao.id = random_id() db.session.merge(user_dao) if flush: db.session.flush() if commit: db.session.commit() return user_dao.to_model()
def save_api_key(api_key, commit=True, flush=False): """ :type api_key: dart.model.api_key.ApiKey """ key_dao = ApiKeyDao() for key, value in user.to_dict().iteritems(): setattr(key_dao, key, value) if not key_dao.id: key_dao.id = random_id() db.session.merge(key_dao) if flush: db.session.flush() if commit: db.session.commit() return key_dao.to_model()
def create_all(): db.create_all() for mutex in Mutexes.all(): sql = """ INSERT INTO mutex (id, version_id, created, updated, name, state) SELECT :id, 0, NOW(), NOW(), :name, :state WHERE NOT EXISTS (SELECT NULL FROM mutex WHERE name = :name) """ statement = text(sql).bindparams(id=random_id(), name=mutex, state=MutexState.READY) db.session.execute(statement) db.session.commit() return 'OK'
def save_event(event, commit=True, flush=False): """ :type event: dart.model.event.Event """ event = default_and_validate(event, event_schema()) event_dao = EventDao() event_dao.id = random_id() event_dao.data = event.data.to_dict() db.session.add(event_dao) if flush: db.session.flush() if commit: db.session.commit() event = event_dao.to_model() return event
def save_workflow(workflow, commit=True, flush=False): """ :type workflow: dart.model.workflow.Workflow """ workflow = default_and_validate(workflow, workflow_schema()) workflow_dao = WorkflowDao() workflow_dao.id = random_id() workflow_dao.data = workflow.data.to_dict() db.session.add(workflow_dao) if flush: db.session.flush() if commit: db.session.commit() workflow = workflow_dao.to_model() return workflow
def save_subgraph_definition(subgraph_definition, engine, trigger_schemas): """ :type engine: dart.model.engine.Engine :type subgraph_definition: dart.model.graph.SubGraphDefinition """ action_schemas = [action_schema(e.params_json_schema) for e in engine.data.supported_action_types] ds_schema = datastore_schema(engine.data.options_json_schema) schema = subgraph_definition_schema(trigger_schemas, action_schemas, ds_schema) subgraph_definition = default_and_validate(subgraph_definition, schema) subgraph_definition_dao = SubGraphDefinitionDao() subgraph_definition_dao.id = random_id() subgraph_definition_dao.data = subgraph_definition.data.to_dict() subgraph_definition_dao.data['engine_name'] = engine.data.name db.session.add(subgraph_definition_dao) db.session.commit() return subgraph_definition_dao.to_model()
def save_event(event, commit=True, flush=False): """ :type event: dart.model.event.Event """ event = default_and_validate(event, event_schema()) event_dao = EventDao() event_dao.id = random_id() event_dao.data = event.data.to_dict() db.session.add(event_dao) if flush: db.session.flush() if commit: db.session.commit() event = event_dao.to_model() return event
def save_workflow(workflow, commit=True, flush=False): """ :type workflow: dart.model.workflow.Workflow """ workflow = default_and_validate(workflow, workflow_schema()) workflow_dao = WorkflowDao() workflow_dao.id = random_id() workflow_dao.data = workflow.data.to_dict() db.session.add(workflow_dao) if flush: db.session.flush() if commit: db.session.commit() workflow = workflow_dao.to_model() return workflow
def save_subscription(self, subscription, commit_and_generate=True, flush=False): """ :type subscription: dart.model.subscription.Subscription """ subscription = default_and_validate(subscription, subscription_schema()) subscription_dao = SubscriptionDao() subscription_dao.id = random_id() subscription.data.state = SubscriptionState.ACTIVE subscription_dao.data = subscription.data.to_dict() db.session.add(subscription_dao) if flush: db.session.flush() subscription = subscription_dao.to_model() if commit_and_generate: db.session.commit() subscription = subscription_dao.to_model() return subscription
def create_all(): db.create_all() for mutex in Mutexes.all(): sql = """ INSERT INTO mutex (id, version_id, created, updated, name, state) SELECT :id, 0, NOW(), NOW(), :name, :state WHERE NOT EXISTS (SELECT NULL FROM mutex WHERE name = :name) """ statement = text(sql).bindparams(id=random_id(), name=mutex, state=MutexState.READY) db.session.execute(statement) db.session.commit() return 'OK'
def save_subscription(self, subscription, commit_and_generate=True, flush=False): """ :type subscription: dart.model.subscription.Subscription """ subscription = default_and_validate(subscription, subscription_schema()) subscription_dao = SubscriptionDao() subscription_dao.id = random_id() subscription.data.state = SubscriptionState.ACTIVE subscription_dao.data = subscription.data.to_dict() db.session.add(subscription_dao) if flush: db.session.flush() subscription = subscription_dao.to_model() if commit_and_generate: db.session.commit() subscription = subscription_dao.to_model() return subscription
def _clone_workflow_action_to_dao(source_action, **data_property_overrides): action = source_action.copy() assert isinstance(action, Action) action.data.workflow_action_id = source_action.id action.data.state = ActionState.HAS_NEVER_RUN action.data.progress = None action.data.queued_time = None action.data.start_time = None action.data.end_time = None action.data.error_message = None for k, v in data_property_overrides.iteritems(): setattr(action.data, k, v) action_dao = ActionDao() action_dao.id = random_id() action_dao.data = action.data.to_dict() return action_dao
def _clone_workflow_action_to_dao(source_action, **data_property_overrides): action = source_action.copy() assert isinstance(action, Action) action.data.workflow_action_id = source_action.id action.data.state = ActionState.PENDING action.data.progress = None action.data.queued_time = None action.data.start_time = None action.data.end_time = None action.data.error_message = None for k, v in data_property_overrides.iteritems(): setattr(action.data, k, v) action_dao = ActionDao() action_dao.id = random_id() action_dao.data = action.data.to_dict() return action_dao
def save_trigger(self, trigger, commit_and_initialize=True, flush=False, user_id=None): wf_uuid = uuid.uuid4().hex # to avoid uuid serialization issues trigger.data.tags = trigger.data.tags if (trigger.data.tags) else [] trigger.data.tags.append(wf_uuid) if user_id: trigger.data.user_id = user_id """ :type trigger: dart.model.trigger.Trigger """ trigger_type_name = trigger.data.trigger_type_name if trigger_type_name == self._manual_trigger_processor.trigger_type( ).name: raise DartValidationException('manual triggers cannot be saved') if trigger_type_name == self._retry_trigger_processor.trigger_type( ).name: raise DartValidationException('retry triggers cannot be saved') trigger_processor = self._trigger_processors.get(trigger_type_name) if not trigger_processor: raise DartValidationException('unknown trigger_type_name: %s' % trigger_type_name) assert isinstance(trigger_processor, TriggerProcessor) trigger = default_and_validate( trigger, trigger_schema( trigger_processor.trigger_type().params_json_schema)) trigger_dao = TriggerDao() trigger_dao.id = random_id() trigger_dao.data = trigger.data.to_dict() db.session.add(trigger_dao) if flush: db.session.flush() trigger = trigger_dao.to_model() if commit_and_initialize: db.session.commit() trigger = trigger_dao.to_model() try: trigger_processor.initialize_trigger(trigger, self) except: db.session.delete(trigger_dao) db.session.commit() raise return trigger
def save_subgraph_definition(subgraph_definition, engine, trigger_schemas): """ :type engine: dart.model.engine.Engine :type subgraph_definition: dart.model.graph.SubGraphDefinition """ action_schemas = [ action_schema(e.params_json_schema) for e in engine.data.supported_action_types ] ds_schema = datastore_schema(engine.data.options_json_schema) schema = subgraph_definition_schema(trigger_schemas, action_schemas, ds_schema) subgraph_definition = default_and_validate(subgraph_definition, schema) subgraph_definition_dao = SubGraphDefinitionDao() subgraph_definition_dao.id = random_id() subgraph_definition_dao.data = subgraph_definition.data.to_dict() subgraph_definition_dao.data['engine_name'] = engine.data.name db.session.add(subgraph_definition_dao) db.session.commit() return subgraph_definition_dao.to_model()
def save_subscription(self, subscription, commit_and_generate=True, flush=False): """ :type subscription: dart.model.subscription.Subscription """ subscription = default_and_validate(subscription, subscription_schema()) subscription_dao = SubscriptionDao() subscription_dao.id = random_id() subscription.data.state = SubscriptionState.QUEUED subscription.data.queued_time = datetime.now() subscription_dao.data = subscription.data.to_dict() db.session.add(subscription_dao) if flush: db.session.flush() subscription = subscription_dao.to_model() if commit_and_generate: db.session.commit() subscription = subscription_dao.to_model() self._subscription_proxy.generate_subscription_elements(subscription) return subscription
def save_trigger(self, trigger, commit_and_initialize=True, flush=False, user_id=None): wf_uuid = uuid.uuid4().hex # to avoid uuid serialization issues trigger.data.tags = trigger.data.tags if (trigger.data.tags) else [] trigger.data.tags.append(wf_uuid) if user_id: trigger.data.user_id = user_id """ :type trigger: dart.model.trigger.Trigger """ trigger_type_name = trigger.data.trigger_type_name if trigger_type_name == self._manual_trigger_processor.trigger_type().name: raise DartValidationException('manual triggers cannot be saved') if trigger_type_name == self._retry_trigger_processor.trigger_type().name: raise DartValidationException('retry triggers cannot be saved') trigger_processor = self._trigger_processors.get(trigger_type_name) if not trigger_processor: raise DartValidationException('unknown trigger_type_name: %s' % trigger_type_name) assert isinstance(trigger_processor, TriggerProcessor) trigger = default_and_validate(trigger, trigger_schema(trigger_processor.trigger_type().params_json_schema)) trigger_dao = TriggerDao() trigger_dao.id = random_id() if trigger_type_name == 'subscription_batch': sub = self._subscription_service.get_subscription(trigger.data.args['subscription_id']) if sub.data.nudge_id: response = self.update_nudge_with_trigger(sub.data.nudge_id, trigger.data.args['unconsumed_data_size_in_bytes'], trigger_dao.id, trigger.data.trigger_type_name) assert(response.status_code == 200) trigger_dao.data = trigger.data.to_dict() db.session.add(trigger_dao) if flush: db.session.flush() trigger = trigger_dao.to_model() if commit_and_initialize: db.session.commit() trigger = trigger_dao.to_model() try: trigger_processor.initialize_trigger(trigger, self) except: db.session.delete(trigger_dao) db.session.commit() raise return trigger
def clone_datastore(self, source_datastore, **data_property_overrides): datastore = Datastore.from_dict(source_datastore.to_dict()) datastore.data.state = DatastoreState.INACTIVE datastore.data.host = None datastore.data.port = None datastore.data.username = None datastore.data.password = None datastore.data.connection_url = None datastore.data.extra_data = None self._set_s3_paths(datastore) for k, v in data_property_overrides.iteritems(): setattr(datastore.data, k, v) datastore_dao = DatastoreDao() datastore_dao.id = random_id() datastore_dao.data = datastore.data.to_dict() db.session.add(datastore_dao) db.session.commit() return datastore_dao.to_model()
def clone_datastore(self, source_datastore, **data_property_overrides): datastore = Datastore.from_dict(source_datastore.to_dict()) datastore.data.state = DatastoreState.INACTIVE datastore.data.host = None datastore.data.port = None datastore.data.username = None datastore.data.password = None datastore.data.connection_url = None datastore.data.extra_data = None self._set_s3_paths(datastore) for k, v in data_property_overrides.iteritems(): setattr(datastore.data, k, v) datastore_dao = DatastoreDao() datastore_dao.id = random_id() datastore_dao.data = datastore.data.to_dict() db.session.add(datastore_dao) db.session.commit() return datastore_dao.to_model()
def save_workflow_instance(workflow, trigger_type, trigger_id, state): """ :type workflow: dart.model.workflow.Workflow :type trigger_type: dart.model.trigger.TriggerType """ wf_instance_dao = WorkflowInstanceDao() wf_instance_dao.id = random_id() wf_data = workflow.data data = WorkflowInstanceData( workflow_id=workflow.id, engine_name=wf_data.engine_name, state=state, trigger_type=trigger_type.name, trigger_id=trigger_id, queued_time=datetime.now(), tags=wf_data.tags, ) wf_instance_dao.data = data.to_dict() db.session.add(wf_instance_dao) db.session.commit() return wf_instance_dao.to_model()
def save_workflow_instance(workflow, trigger_type, trigger_id, state): """ :type workflow: dart.model.workflow.Workflow :type trigger_type: dart.model.trigger.TriggerType """ wf_instance_dao = WorkflowInstanceDao() wf_instance_dao.id = random_id() wf_data = workflow.data data = WorkflowInstanceData( workflow_id=workflow.id, engine_name=wf_data.engine_name, state=state, trigger_type=trigger_type.name, trigger_id=trigger_id, queued_time=datetime.now(), tags=wf_data.tags, ) wf_instance_dao.data = data.to_dict() db.session.add(wf_instance_dao) db.session.commit() return wf_instance_dao.to_model()
def to_entity_models_with_randomized_ids(entity_models): # To support editing multiple subgraphs concurrently, we need to randomize the ids. # This method takes advantage of JSON formatting (a bit hacky, but simple and fast). id_map = {} for e in entity_models: prefix = 'UNSAVED' if e.id.startswith('PARENT'): prefix = 'PARENT' if e.id.startswith('CHILD'): prefix = 'CHILD' id_map[e.id] = '%s-%s' % (prefix, random_id()) results = [] for em in entity_models: stringified_model = json.dumps(em.to_dict()) for original_id, randomized_id in id_map.iteritems(): stringified_model = stringified_model.replace('"%s"' % original_id, '"%s"' % randomized_id) cls = type(em) results.append(cls.from_dict(json.loads(stringified_model))) return results
def save_engine(self, engine): """ :type engine: dart.model.engine.Engine """ engine = default_and_validate(engine, engine_schema()) self._validate_ecs_task_definition(engine.data.ecs_task_definition) engine_dao = EngineDao() engine_dao.id = random_id() engine_dao.name = engine.data.name engine_dao.data = engine.data.to_dict() db.session.add(engine_dao) try: db.session.commit() engine = engine_dao.to_model() engine.data.ecs_task_definition_arn = self._register_ecs_task_definition(engine) return self.update_engine_data(engine.id, engine.data) except SqlAlchemyIntegrityError as e: if hasattr(e, 'orig') and isinstance(e.orig, PostgresIntegrityError) and e.orig.pgcode == '23505': raise DartValidationException('name already exists: %s' % engine.data.name) raise e
def to_entity_models_with_randomized_ids(entity_models): # To support editing multiple subgraphs concurrently, we need to randomize the ids. # This method takes advantage of JSON formatting (a bit hacky, but simple and fast). id_map = {} for e in entity_models: prefix = 'UNSAVED' if e.id.startswith('PARENT'): prefix = 'PARENT' if e.id.startswith('CHILD'): prefix = 'CHILD' id_map[e.id] = '%s-%s' % (prefix, random_id()) results = [] for em in entity_models: stringified_model = json.dumps(em.to_dict()) for original_id, randomized_id in id_map.iteritems(): stringified_model = stringified_model.replace('"%s"' % original_id, '"%s"' % randomized_id) cls = type(em) results.append(cls.from_dict(json.loads(stringified_model))) return results
def save_dataset(dataset, commit=True, flush=False): """ :type dataset: dart.model.dataset.Dataset """ dataset = default_and_validate(dataset, dataset_schema()) dataset_dao = DatasetDao() dataset_dao.id = random_id() dataset_dao.name = dataset.data.name dataset.data.location = dataset.data.location.rstrip('/') dataset_dao.data = dataset.data.to_dict() db.session.add(dataset_dao) try: if flush: db.session.flush() if commit: db.session.commit() dataset = dataset_dao.to_model() return dataset except SqlAlchemyIntegrityError as e: if hasattr(e, 'orig') and isinstance(e.orig, PostgresIntegrityError) and e.orig.pgcode == '23505': raise DartValidationException('name already exists: %s' % dataset.data.name) raise e
def save_subscription(self, subscription, commit_and_generate=True, flush=False): """ :type subscription: dart.model.subscription.Subscription """ subscription = default_and_validate(subscription, subscription_schema()) subscription_dao = SubscriptionDao() subscription_dao.id = random_id() subscription.data.state = SubscriptionState.QUEUED subscription.data.queued_time = datetime.now() subscription_dao.data = subscription.data.to_dict() db.session.add(subscription_dao) if flush: db.session.flush() subscription = subscription_dao.to_model() if commit_and_generate: db.session.commit() subscription = subscription_dao.to_model() self._subscription_proxy.generate_subscription_elements( subscription) return subscription
def save_trigger(self, trigger, commit_and_initialize=True, flush=False, user_id=None): wf_uuid = uuid.uuid4().hex # to avoid uuid serialization issues trigger.data.tags = trigger.data.tags if (trigger.data.tags) else [] trigger.data.tags.append(wf_uuid) if user_id: trigger.data.user_id = user_id """ :type trigger: dart.model.trigger.Trigger """ trigger_type_name = trigger.data.trigger_type_name if trigger_type_name == self._manual_trigger_processor.trigger_type().name: raise DartValidationException('manual triggers cannot be saved') if trigger_type_name == self._retry_trigger_processor.trigger_type().name: raise DartValidationException('retry triggers cannot be saved') trigger_processor = self._trigger_processors.get(trigger_type_name) if not trigger_processor: raise DartValidationException('unknown trigger_type_name: %s' % trigger_type_name) assert isinstance(trigger_processor, TriggerProcessor) trigger = default_and_validate(trigger, trigger_schema(trigger_processor.trigger_type().params_json_schema)) trigger_dao = TriggerDao() trigger_dao.id = random_id() trigger_dao.data = trigger.data.to_dict() db.session.add(trigger_dao) if flush: db.session.flush() trigger = trigger_dao.to_model() if commit_and_initialize: db.session.commit() trigger = trigger_dao.to_model() try: trigger_processor.initialize_trigger(trigger, self) except: db.session.delete(trigger_dao) db.session.commit() raise return trigger
def _handle_docker_concerns(self, cwl_image, eng_cfg, misc_log_group_name, output_config, syslog_log_group_name): if 'docker' in self.stacks_to_skip: _logger.info('skipping docker concerns') return _logger.info('configuring and building cloudwatch logs docker image (a special snowflake)') dart_root = dart_root_relative_path() r_id = random_id() values = (dart_root, r_id) call('cd %s && cd .. && git clone https://github.com/awslabs/ecs-cloudwatch-logs dart-cwl-%s' % values) docker_init = dart_root_relative_path('tools', 'docker', 'docker-local-init.sh') with open(dart_root_relative_path('aws', 'cloudwatch-logs', 'awslogs_template.conf')) as cwl_conf_template, \ open(dart_root_relative_path('..', 'dart-cwl-%s/awslogs.conf' % r_id), mode='w') as cwl_conf: contents = cwl_conf_template.read() contents = contents.replace('{DART_LOG_GROUP_SYSLOG}', syslog_log_group_name) contents = contents.replace('{DART_LOG_GROUP_MISC}', misc_log_group_name) cwl_conf.write(contents) cwl_root = dart_root_relative_path('..', 'dart-cwl-%s' % r_id) call('source %s && cd %s && docker build -f Dockerfile -t %s .' % (docker_init, cwl_root, cwl_image)) _logger.info('running grunt build') call('cd %s && grunt build' % dart_root_relative_path('src', 'python', 'dart', 'web', 'ui')) _logger.info('building other docker images') for repo_name in [rn for rn in output_config['ecr']['repo_names'] if not rn.endswith('cloudwatchlogs')]: version = eng_cfg['emr_engine']['options']['impala_version'] if 'impala' in repo_name else '1.0.0' docker_img = self._docker_image(repo_name, output_config, version=version) docker_file_suffix = repo_name.split('/')[-1] values = (docker_init, dart_root, docker_file_suffix, docker_img) call('source %s && cd %s && docker build -f tools/docker/Dockerfile-%s -t %s .' % values) _logger.info('pushing docker images') cmd = ('source %s && cd %s && $(aws ecr get-login)' % (docker_init, dart_root)) + ' && docker push %s' for repo_name in output_config['ecr']['repo_names']: version = eng_cfg['emr_engine']['options']['impala_version'] if 'impala' in repo_name else '1.0.0' call(cmd % self._docker_image(repo_name, output_config, version=version))
def save_engine(self, engine): """ :type engine: dart.model.engine.Engine """ engine = default_and_validate(engine, engine_schema()) self._validate_ecs_task_definition(engine.data.ecs_task_definition) engine_dao = EngineDao() engine_dao.id = random_id() engine_dao.name = engine.data.name engine_dao.data = engine.data.to_dict() db.session.add(engine_dao) try: db.session.commit() engine = engine_dao.to_model() engine.data.ecs_task_definition_arn = self._register_ecs_task_definition( engine) return self.update_engine_data(engine.id, engine.data) except SqlAlchemyIntegrityError as e: if hasattr(e, 'orig') and isinstance( e.orig, PostgresIntegrityError) and e.orig.pgcode == '23505': raise DartValidationException('name already exists: %s' % engine.data.name) raise e
def save_dataset(dataset, commit=True, flush=False): """ :type dataset: dart.model.dataset.Dataset """ dataset = default_and_validate(dataset, dataset_schema()) dataset_dao = DatasetDao() dataset_dao.id = random_id() dataset_dao.name = dataset.data.name dataset.data.location = dataset.data.location.rstrip('/') dataset_dao.data = dataset.data.to_dict() db.session.add(dataset_dao) try: if flush: db.session.flush() if commit: db.session.commit() dataset = dataset_dao.to_model() return dataset except SqlAlchemyIntegrityError as e: if hasattr(e, 'orig') and isinstance( e.orig, PostgresIntegrityError) and e.orig.pgcode == '23505': raise DartValidationException('name already exists: %s' % dataset.data.name) raise e
def setUp(self): dart = Dart(host='localhost', port=5000) """ :type dart: dart.client.python.dart_client.Dart """ self.dart = dart cs = [Column('c1', DataType.VARCHAR, 50), Column('c2', DataType.BIGINT)] df = DataFormat(FileFormat.PARQUET, RowFormat.NONE) dataset_data = DatasetData('test-dataset0', 'test_dataset_table0', 's3://test/dataset/0/%s' + random_id(), df, cs) self.dataset0 = self.dart.save_dataset(Dataset(data=dataset_data)) cs = [Column('c1', DataType.VARCHAR, 50), Column('c2', DataType.BIGINT)] df = DataFormat(FileFormat.PARQUET, RowFormat.NONE) dataset1_location = 's3://test/dataset/1/%s' + random_id() dataset_data = DatasetData('test-dataset1', 'test_dataset_table1', dataset1_location, df, cs) self.dataset1 = self.dart.save_dataset(Dataset(data=dataset_data)) cs = [Column('c1', DataType.VARCHAR, 50), Column('c2', DataType.BIGINT)] df = DataFormat(FileFormat.PARQUET, RowFormat.NONE) dataset_data = DatasetData('test-dataset2-no-show', 'test_dataset_table2', 's3://test/dataset/2/%s' + random_id(), df, cs) self.dataset2 = self.dart.save_dataset(Dataset(data=dataset_data)) s = Subscription(data=SubscriptionData('test-subscription0', self.dataset0.id)) self.subscription0 = self.dart.save_subscription(s) s = Subscription(data=SubscriptionData('test-subscription2-no-show', self.dataset2.id)) self.subscription2 = self.dart.save_subscription(s) dst_args = {'action_sleep_time_in_seconds': 0} dst = Datastore(data=DatastoreData('test-datastore0', 'no_op_engine', args=dst_args, state=DatastoreState.TEMPLATE)) self.datastore0 = self.dart.save_datastore(dst) dst = Datastore(data=DatastoreData('test-datastore1', 'no_op_engine', args=dst_args, state=DatastoreState.TEMPLATE)) self.datastore1 = self.dart.save_datastore(dst) dst = Datastore(data=DatastoreData('test-datastore2-no-show', 'no_op_engine', args=dst_args, state=DatastoreState.ACTIVE)) self.datastore2 = self.dart.save_datastore(dst) wf0 = Workflow(data=WorkflowData('test-workflow0', self.datastore0.id, state=WorkflowState.ACTIVE)) self.workflow0 = self.dart.save_workflow(wf0, self.datastore0.id) wf1 = Workflow(data=WorkflowData('test-workflow1', self.datastore1.id, state=WorkflowState.ACTIVE)) self.workflow1 = self.dart.save_workflow(wf1, self.datastore1.id) wf2 = Workflow(data=WorkflowData('test-workflow2-no-show', self.datastore2.id, state=WorkflowState.ACTIVE)) self.workflow2 = self.dart.save_workflow(wf2, self.datastore2.id) a_args = {'source_hdfs_path': 'hdfs:///user/hive/warehouse/test', 'destination_s3_path': dataset1_location} a00 = Action(data=ActionData(NoOpActionTypes.action_that_succeeds.name, NoOpActionTypes.action_that_succeeds.name, state=ActionState.TEMPLATE)) a01 = Action(data=ActionData(NoOpActionTypes.consume_subscription.name, NoOpActionTypes.consume_subscription.name, {'subscription_id': self.subscription0.id}, state=ActionState.TEMPLATE)) a02 = Action(data=ActionData(NoOpActionTypes.action_that_succeeds.name, NoOpActionTypes.action_that_succeeds.name, state=ActionState.TEMPLATE)) a03 = Action(data=ActionData(NoOpActionTypes.copy_hdfs_to_s3_action.name, NoOpActionTypes.copy_hdfs_to_s3_action.name, a_args, state=ActionState.TEMPLATE)) a04 = Action(data=ActionData(NoOpActionTypes.action_that_succeeds.name, NoOpActionTypes.action_that_succeeds.name, state=ActionState.TEMPLATE)) self.action00, self.action01, self.action02, self.action03, self.action04 = \ self.dart.save_actions([a00, a01, a02, a03, a04], workflow_id=self.workflow0.id) a10 = Action(data=ActionData(NoOpActionTypes.load_dataset.name, NoOpActionTypes.load_dataset.name, {'dataset_id': self.dataset1.id}, state=ActionState.TEMPLATE)) self.action10 = self.dart.save_actions([a10], workflow_id=self.workflow1.id) a20 = Action(data=ActionData(NoOpActionTypes.action_that_succeeds.name, NoOpActionTypes.action_that_succeeds.name, state=ActionState.HAS_NEVER_RUN)) a21 = Action(data=ActionData(NoOpActionTypes.load_dataset.name, NoOpActionTypes.load_dataset.name, {'dataset_id': self.dataset2.id}, state=ActionState.TEMPLATE)) self.action20 = self.dart.save_actions([a20], datastore_id=self.datastore2.id) self.action21 = self.dart.save_actions([a21], workflow_id=self.workflow2.id) self.event1 = self.dart.save_event(Event(data=EventData('test-event1', state=EventState.ACTIVE))) self.event2 = self.dart.save_event(Event(data=EventData('test-event2-no-show', state=EventState.ACTIVE))) tr_args = {'event_id': self.event1.id} tr = Trigger(data=TriggerData('test-event-trigger1', 'event', [self.workflow1.id], tr_args, TriggerState.ACTIVE)) self.event_trigger1 = self.dart.save_trigger(tr) tr_args = {'event_id': self.event2.id} tr = Trigger(data=TriggerData('test-event-trigger2-no-show', 'event', [self.workflow2.id], tr_args, TriggerState.ACTIVE)) self.event_trigger2 = self.dart.save_trigger(tr) st_args = {'fire_after': 'ALL', 'completed_trigger_ids': [self.event_trigger1.id]} st = Trigger(data=TriggerData('test-super-trigger1', 'super', None, st_args, TriggerState.ACTIVE)) self.super_trigger1 = self.dart.save_trigger(st) st_args = {'fire_after': 'ANY', 'completed_trigger_ids': [self.super_trigger1.id]} st = Trigger(data=TriggerData('test-super-trigger2', 'super', [self.workflow1.id], st_args, TriggerState.ACTIVE)) self.super_trigger2 = self.dart.save_trigger(st)