def get_action_json_schema(action_name): action_id = request.args.get('action_id') datastore_id = request.args.get('datastore_id') workflow_id = request.args.get('workflow_id') engine_name = request.args.get('engine_name') if action_id: action = action_service().get_action(action_id, raise_when_missing=False) if not action: return {'results': 'ERROR', 'error_message': 'no action with id: %s' % action_id}, 404, None engine_name = action.data.engine_name if workflow_id: workflow = workflow_service().get_workflow(workflow_id, raise_when_missing=False) if not workflow: return {'results': 'ERROR', 'error_message': 'no workflow with id: %s' % workflow_id}, 404, None engine_name = workflow.data.engine_name if datastore_id: datastore = datastore_service().get_datastore(datastore_id, raise_when_missing=False) if not datastore: return {'results': 'ERROR', 'error_message': 'no datastore with id: %s' % datastore_id}, 404, None engine_name = datastore.data.engine_name if engine_name: engine = engine_service().get_engine_by_name(engine_name) if not engine: return {'results': 'ERROR', 'error_message': 'unknown engine with name: %s' % engine_name}, 400, None for action_type in engine.data.supported_action_types: if action_type.name == action_name: return {'results': action_schema(action_type.params_json_schema)} return {'results': 'ERROR', 'error_message': 'unknown action with name: %s' % action_name}, 400, None return {'results': 'ERROR', 'error_message': 'one of datastore_id or workflow_id must be provided'}, 400, None
def test_action_schema_invalid(self): with self.assertRaises(DartValidationException) as context: a = Action(data=ActionData('copy_hdfs_to_s3', 'copy_hdfs_to_s3', { 'source_hdfs_path': 'hdfs:///user/hive/warehouse/dtest4', # 'destination_s3_path': 's3://fake-bucket/dart_testing', }, engine_name='no_op_engine')) # should fail because destination_s3_path is required default_and_validate(a, action_schema(NoOpActionTypes.copy_hdfs_to_s3_action.params_json_schema)).to_dict() self.assertTrue(isinstance(context.exception, DartValidationException))
def default_and_validate_action(self, action, action_type=None): if not action_type: engine = self._engine_service.get_engine_by_name(action.data.engine_name) action_types_by_name = {at.name: at for at in engine.data.supported_action_types} action_type = action_types_by_name.get(action.data.action_type_name) if not action_type: raise DartValidationException('unknown action: "%s"' % action.data.action_type_name) assert isinstance(action_type, ActionType) action = default_and_validate(action, action_schema(action_type.params_json_schema)) return action
def test_action_schema(self): last_in_workflow = None a = Action(data=ActionData('copy_hdfs_to_s3', 'copy_hdfs_to_s3', { 'source_hdfs_path': 'hdfs:///user/hive/warehouse/dtest4', 'destination_s3_path': 's3://fake-bucket/dart_testing', }, engine_name='no_op_engine', last_in_workflow=last_in_workflow)) obj_before = a.to_dict() obj_after = default_and_validate(a, action_schema(NoOpActionTypes.copy_hdfs_to_s3_action.params_json_schema)).to_dict() # many fields should have been defaulted, making these unequal self.assertNotEqual(obj_before, obj_after)
def default_and_validate_action(self, action, action_type=None): if not action_type: engine = self._engine_service.get_engine_by_name(action.data.engine_name) action_types_by_name = {at.name: at for at in engine.data.supported_action_types} action_type = action_types_by_name.get(action.data.action_type_name) if not action_type: raise DartValidationException('unknown action: "%s"' % action.data.action_type_name) assert isinstance(action_type, ActionType) if not action.data.args: action.data.args = {} action = default_and_validate(action, action_schema(action_type.params_json_schema)) return action
def save_subgraph_definition(subgraph_definition, engine, trigger_schemas): """ :type engine: dart.model.engine.Engine :type subgraph_definition: dart.model.graph.SubGraphDefinition """ action_schemas = [action_schema(e.params_json_schema) for e in engine.data.supported_action_types] ds_schema = datastore_schema(engine.data.options_json_schema) schema = subgraph_definition_schema(trigger_schemas, action_schemas, ds_schema) subgraph_definition = default_and_validate(subgraph_definition, schema) subgraph_definition_dao = SubGraphDefinitionDao() subgraph_definition_dao.id = random_id() subgraph_definition_dao.data = subgraph_definition.data.to_dict() subgraph_definition_dao.data['engine_name'] = engine.data.name db.session.add(subgraph_definition_dao) db.session.commit() return subgraph_definition_dao.to_model()
def get_action_json_schema_empty(): supported_action_type_params_schema = None action_id = request.args.get('action_id') if action_id: action = action_service().get_action(action_id, raise_when_missing=False) if not action: return {'results': 'ERROR', 'error_message': 'no action with id: %s' % action_id}, 404, None engine_name = action.data.engine_name engine = engine_service().get_engine_by_name(engine_name) for action_type in engine.data.supported_action_types: if action_type.name == action.data.action_type_name: supported_action_type_params_schema = action_type.params_json_schema break return {'results': action_schema(supported_action_type_params_schema)}
def _query_action_query(self, filters, order_by=None): """ :type filters: list[dart.model.query.Filter] :type order_by: list[dart.model.query.OrderBy] """ action_types = [] for engine in self._engine_service.query_engines([], 1000, 0): action_types.extend(engine.data.supported_action_types) action_schemas = [action_schema(a.params_json_schema) for a in action_types] query = ActionDao.query for o in (order_by or []): query = self._order_by_service.apply_order_by(o, query, ActionDao, action_schemas) for f in filters: query = self._filter_service.apply_filter(f, query, ActionDao, action_schemas) return query
def test_action_schema(self): last_in_workflow = None a = Action(data=ActionData( 'copy_hdfs_to_s3', 'copy_hdfs_to_s3', { 'source_hdfs_path': 'hdfs:///user/hive/warehouse/dtest4', 'destination_s3_path': 's3://fake-bucket/dart_testing', }, engine_name='no_op_engine', last_in_workflow=last_in_workflow)) obj_before = a.to_dict() obj_after = default_and_validate( a, action_schema(NoOpActionTypes.copy_hdfs_to_s3_action. params_json_schema)).to_dict() # many fields should have been defaulted, making these unequal self.assertNotEqual(obj_before, obj_after)
def test_action_schema_invalid(self): with self.assertRaises(DartValidationException) as context: a = Action(data=ActionData( 'copy_hdfs_to_s3', 'copy_hdfs_to_s3', { 'source_hdfs_path': 'hdfs:///user/hive/warehouse/dtest4', # 'destination_s3_path': 's3://fake-bucket/dart_testing', }, engine_name='no_op_engine')) # should fail because destination_s3_path is required default_and_validate( a, action_schema(NoOpActionTypes.copy_hdfs_to_s3_action. params_json_schema)).to_dict() self.assertTrue(isinstance(context.exception, DartValidationException))
def save_subgraph_definition(subgraph_definition, engine, trigger_schemas): """ :type engine: dart.model.engine.Engine :type subgraph_definition: dart.model.graph.SubGraphDefinition """ action_schemas = [ action_schema(e.params_json_schema) for e in engine.data.supported_action_types ] ds_schema = datastore_schema(engine.data.options_json_schema) schema = subgraph_definition_schema(trigger_schemas, action_schemas, ds_schema) subgraph_definition = default_and_validate(subgraph_definition, schema) subgraph_definition_dao = SubGraphDefinitionDao() subgraph_definition_dao.id = random_id() subgraph_definition_dao.data = subgraph_definition.data.to_dict() subgraph_definition_dao.data['engine_name'] = engine.data.name db.session.add(subgraph_definition_dao) db.session.commit() return subgraph_definition_dao.to_model()
def main(): data = { 'definitions': { 'Action': action_schema(None), 'ActionContext': action_context_schema(), 'ActionResult': action_result_schema(), 'Dataset': dataset_schema(), 'Datastore': datastore_schema(None), 'Engine': engine_schema(), 'ErrorResult': error_result_schema(), 'Event': event_schema(), 'Filter': filter_schema(), 'GraphEntityIdentifier': graph_entity_identifier_schema(), 'GraphEntity': graph_entity_schema(), 'JSONPatch': json_patch_schema(), 'JSONSchema': json_schema_schema(), 'OKResult': ok_result_schema(), 'OrderBy': order_by_schema(), 'SubGraph': sub_graph_schema(), 'SubGraphDefinition': { 'type': 'object' }, #subgraph_definition_schema([{'type': 'object'}], [{'type': 'object'}], {'type': 'object'}), 'Subscription': subscription_schema(), 'Trigger': trigger_schema({'type': 'object'}), 'TriggerType': trigger_type_schema(), 'Workflow': workflow_schema(), 'WorkflowInstance': workflow_instance_schema() } } fix_up(data, data, [None]) print dump(data, Dumper=Dumper, default_style=None, default_flow_style=False, explicit_start=False, explicit_end=False) return 0
def export_swagger_definitions(out): data = { 'definitions': { 'Action': action_schema({'type': 'object', 'x-nullable': True}), 'ActionResponse': object_response_schema('Action'), 'ActionsResponse': array_response_schema('Action'), 'PagedActionsResponse': paged_response_schema('Action'), 'ActionContext': action_context_schema(), 'ActionContextResponse': object_response_schema('ActionContext'), 'ActionResult': action_result_schema(), 'Dataset': dataset_schema(), 'DatasetResponse': object_response_schema('Dataset'), 'PagedDatasetsResponse': paged_response_schema('Dataset'), 'Datastore': datastore_schema({'type': 'object', 'x-nullable': True}), 'DatastoreResponse': object_response_schema('Datastore'), 'PagedDatastoresResponse': paged_response_schema('Datastore'), 'Engine': engine_schema(), 'EngineResponse': object_response_schema('Engine'), 'PagedEnginesResponse': paged_response_schema('Engine'), 'ErrorResponse': error_response_schema(), 'Event': event_schema(), 'EventResponse': object_response_schema('Event'), 'PagedEventsResponse': paged_response_schema('Event'), 'Filter': filter_schema(), 'GraphEntityIdentifier': graph_entity_identifier_schema(), 'GraphEntityIdentifierResponse': object_response_schema('GraphEntityIdentifier'), 'GraphEntityIdentifiersResponse': array_response_schema('GraphEntityIdentifier'), 'GraphEntity': graph_entity_schema(), 'GraphEntityResponse': object_response_schema('GraphEntity'), 'JSONPatch': json_patch_schema(), 'JSONSchema': json_schema_schema(), 'JSONSchemaResponse': object_response_schema('JSONSchema'), 'ObjectResponse': object_response_schema('object'), 'ObjectsResponse': array_response_schema('object'), 'PagedObjectsResponse': paged_response_schema('object'), 'OKResponse': ok_response_schema(), 'OrderBy': order_by_schema(), 'Subgraph': sub_graph_schema(), 'SubgraphResponse': object_response_schema('Subgraph'), 'SubgraphDefinition': {'type': 'object'}, #subgraph_definition_schema([{'type': 'object'}], [{'type': 'object'}], {'type': 'object'}), 'SubgraphDefinitionResponse': object_response_schema('SubgraphDefinition'), 'Subscription': subscription_schema(), 'SubscriptionResponse': object_response_schema('Subscription'), 'PagedSubscriptionsResponse': paged_response_schema('Subscription'), 'SubscriptionElement': subscription_element_schema(), 'PagedSubscriptionElementsResponse': paged_response_schema('SubscriptionElement'), 'Trigger': trigger_schema({'type': 'object'}), 'TriggerResponse': object_response_schema('Trigger'), 'PagedTriggersResponse': paged_response_schema('Trigger'), 'TriggerType': trigger_type_schema(), 'PagedTriggerTypesResponse': paged_response_schema('TriggerType'), 'Workflow': workflow_schema(), 'WorkflowResponse': object_response_schema('Workflow'), 'PagedWorkflowsResponse': paged_response_schema('Workflow'), 'WorkflowInstance': workflow_instance_schema(), 'WorkflowInstanceResponse': object_response_schema('WorkflowInstance'), 'PagedWorkflowInstancesResponse': paged_response_schema('WorkflowInstance') } } fix_up(data, data, [None]) dump(data, out, Dumper=Dumper, default_style=None, default_flow_style=False, explicit_start=False, explicit_end=False)
def export_swagger_definitions(out): data = { 'definitions': { 'Action': action_schema({ 'type': 'object', 'x-nullable': True }), 'ActionResponse': object_response_schema('Action'), 'ActionsResponse': array_response_schema('Action'), 'PagedActionsResponse': paged_response_schema('Action'), 'ActionContext': action_context_schema(), 'ActionContextResponse': object_response_schema('ActionContext'), 'ActionResult': action_result_schema(), 'Dataset': dataset_schema(), 'DatasetResponse': object_response_schema('Dataset'), 'PagedDatasetsResponse': paged_response_schema('Dataset'), 'Datastore': datastore_schema({ 'type': 'object', 'x-nullable': True }), 'DatastoreResponse': object_response_schema('Datastore'), 'PagedDatastoresResponse': paged_response_schema('Datastore'), 'Engine': engine_schema(), 'EngineResponse': object_response_schema('Engine'), 'PagedEnginesResponse': paged_response_schema('Engine'), 'ErrorResponse': error_response_schema(), 'Event': event_schema(), 'EventResponse': object_response_schema('Event'), 'PagedEventsResponse': paged_response_schema('Event'), 'Filter': filter_schema(), 'GraphEntityIdentifier': graph_entity_identifier_schema(), 'GraphEntityIdentifierResponse': object_response_schema('GraphEntityIdentifier'), 'GraphEntityIdentifiersResponse': array_response_schema('GraphEntityIdentifier'), 'GraphEntity': graph_entity_schema(), 'GraphEntityResponse': object_response_schema('GraphEntity'), 'JSONPatch': json_patch_schema(), 'JSONSchema': json_schema_schema(), 'JSONSchemaResponse': object_response_schema('JSONSchema'), 'ObjectResponse': object_response_schema('object'), 'ObjectsResponse': array_response_schema('object'), 'PagedObjectsResponse': paged_response_schema('object'), 'OKResponse': ok_response_schema(), 'OrderBy': order_by_schema(), 'Subgraph': sub_graph_schema(), 'SubgraphResponse': object_response_schema('Subgraph'), 'SubgraphDefinition': { 'type': 'object' }, #subgraph_definition_schema([{'type': 'object'}], [{'type': 'object'}], {'type': 'object'}), 'SubgraphDefinitionResponse': object_response_schema('SubgraphDefinition'), 'Subscription': subscription_schema(), 'SubscriptionResponse': object_response_schema('Subscription'), 'PagedSubscriptionsResponse': paged_response_schema('Subscription'), 'SubscriptionElement': subscription_element_schema(), 'PagedSubscriptionElementsResponse': paged_response_schema('SubscriptionElement'), 'Trigger': trigger_schema({'type': 'object'}), 'TriggerResponse': object_response_schema('Trigger'), 'PagedTriggersResponse': paged_response_schema('Trigger'), 'TriggerType': trigger_type_schema(), 'PagedTriggerTypesResponse': paged_response_schema('TriggerType'), 'Workflow': workflow_schema(), 'WorkflowResponse': object_response_schema('Workflow'), 'PagedWorkflowsResponse': paged_response_schema('Workflow'), 'WorkflowInstance': workflow_instance_schema(), 'WorkflowInstanceResponse': object_response_schema('WorkflowInstance'), 'PagedWorkflowInstancesResponse': paged_response_schema('WorkflowInstance') } } fix_up(data, data, [None]) dump(data, out, Dumper=Dumper, default_style=None, default_flow_style=False, explicit_start=False, explicit_end=False)