def get_action_json_schema(action_name):
action_id = request.args.get('action_id')
datastore_id = request.args.get('datastore_id')
workflow_id = request.args.get('workflow_id')
engine_name = request.args.get('engine_name')
if action_id:
action = action_service().get_action(action_id, raise_when_missing=False)
if not action:
return {'results': 'ERROR', 'error_message': 'no action with id: %s' % action_id}, 404, None
engine_name = action.data.engine_name
if workflow_id:
workflow = workflow_service().get_workflow(workflow_id, raise_when_missing=False)
if not workflow:
return {'results': 'ERROR', 'error_message': 'no workflow with id: %s' % workflow_id}, 404, None
engine_name = workflow.data.engine_name
if datastore_id:
datastore = datastore_service().get_datastore(datastore_id, raise_when_missing=False)
if not datastore:
return {'results': 'ERROR', 'error_message': 'no datastore with id: %s' % datastore_id}, 404, None
engine_name = datastore.data.engine_name
if engine_name:
engine = engine_service().get_engine_by_name(engine_name)
if not engine:
return {'results': 'ERROR', 'error_message': 'unknown engine with name: %s' % engine_name}, 400, None
for action_type in engine.data.supported_action_types:
if action_type.name == action_name:
return {'results': action_schema(action_type.params_json_schema)}
return {'results': 'ERROR', 'error_message': 'unknown action with name: %s' % action_name}, 400, None
return {'results': 'ERROR', 'error_message': 'one of datastore_id or workflow_id must be provided'}, 400, None
def get_action_json_schema(action_name):
action_id = request.args.get('action_id')
datastore_id = request.args.get('datastore_id')
workflow_id = request.args.get('workflow_id')
engine_name = request.args.get('engine_name')
if action_id:
action = action_service().get_action(action_id, raise_when_missing=False)
if not action:
return {'results': 'ERROR', 'error_message': 'no action with id: %s' % action_id}, 404, None
engine_name = action.data.engine_name
if workflow_id:
workflow = workflow_service().get_workflow(workflow_id, raise_when_missing=False)
if not workflow:
return {'results': 'ERROR', 'error_message': 'no workflow with id: %s' % workflow_id}, 404, None
engine_name = workflow.data.engine_name
if datastore_id:
datastore = datastore_service().get_datastore(datastore_id, raise_when_missing=False)
if not datastore:
return {'results': 'ERROR', 'error_message': 'no datastore with id: %s' % datastore_id}, 404, None
engine_name = datastore.data.engine_name
if engine_name:
engine = engine_service().get_engine_by_name(engine_name)
if not engine:
return {'results': 'ERROR', 'error_message': 'unknown engine with name: %s' % engine_name}, 400, None
for action_type in engine.data.supported_action_types:
if action_type.name == action_name:
return {'results': action_schema(action_type.params_json_schema)}
return {'results': 'ERROR', 'error_message': 'unknown action with name: %s' % action_name}, 400, None
return {'results': 'ERROR', 'error_message': 'one of datastore_id or workflow_id must be provided'}, 400, None
def test_action_schema_invalid(self):
with self.assertRaises(DartValidationException) as context:
a = Action(data=ActionData('copy_hdfs_to_s3', 'copy_hdfs_to_s3', {
'source_hdfs_path': 'hdfs:///user/hive/warehouse/dtest4',
# 'destination_s3_path': 's3://fake-bucket/dart_testing',
}, engine_name='no_op_engine'))
# should fail because destination_s3_path is required
default_and_validate(a, action_schema(NoOpActionTypes.copy_hdfs_to_s3_action.params_json_schema)).to_dict()
self.assertTrue(isinstance(context.exception, DartValidationException))
def default_and_validate_action(self, action, action_type=None):
if not action_type:
engine = self._engine_service.get_engine_by_name(action.data.engine_name)
action_types_by_name = {at.name: at for at in engine.data.supported_action_types}
action_type = action_types_by_name.get(action.data.action_type_name)
if not action_type:
raise DartValidationException('unknown action: "%s"' % action.data.action_type_name)
assert isinstance(action_type, ActionType)
action = default_and_validate(action, action_schema(action_type.params_json_schema))
return action
def test_action_schema(self):
last_in_workflow = None
a = Action(data=ActionData('copy_hdfs_to_s3', 'copy_hdfs_to_s3', {
'source_hdfs_path': 'hdfs:///user/hive/warehouse/dtest4',
'destination_s3_path': 's3://fake-bucket/dart_testing',
}, engine_name='no_op_engine', last_in_workflow=last_in_workflow))
obj_before = a.to_dict()
obj_after = default_and_validate(a, action_schema(NoOpActionTypes.copy_hdfs_to_s3_action.params_json_schema)).to_dict()
# many fields should have been defaulted, making these unequal
self.assertNotEqual(obj_before, obj_after)
def default_and_validate_action(self, action, action_type=None):
if not action_type:
engine = self._engine_service.get_engine_by_name(action.data.engine_name)
action_types_by_name = {at.name: at for at in engine.data.supported_action_types}
action_type = action_types_by_name.get(action.data.action_type_name)
if not action_type:
raise DartValidationException('unknown action: "%s"' % action.data.action_type_name)
assert isinstance(action_type, ActionType)
if not action.data.args:
action.data.args = {}
action = default_and_validate(action, action_schema(action_type.params_json_schema))
return action
def save_subgraph_definition(subgraph_definition, engine, trigger_schemas):
""" :type engine: dart.model.engine.Engine
:type subgraph_definition: dart.model.graph.SubGraphDefinition """
action_schemas = [action_schema(e.params_json_schema) for e in engine.data.supported_action_types]
ds_schema = datastore_schema(engine.data.options_json_schema)
schema = subgraph_definition_schema(trigger_schemas, action_schemas, ds_schema)
subgraph_definition = default_and_validate(subgraph_definition, schema)
subgraph_definition_dao = SubGraphDefinitionDao()
subgraph_definition_dao.id = random_id()
subgraph_definition_dao.data = subgraph_definition.data.to_dict()
subgraph_definition_dao.data['engine_name'] = engine.data.name
db.session.add(subgraph_definition_dao)
db.session.commit()
return subgraph_definition_dao.to_model()
def get_action_json_schema_empty():
supported_action_type_params_schema = None
action_id = request.args.get('action_id')
if action_id:
action = action_service().get_action(action_id, raise_when_missing=False)
if not action:
return {'results': 'ERROR', 'error_message': 'no action with id: %s' % action_id}, 404, None
engine_name = action.data.engine_name
engine = engine_service().get_engine_by_name(engine_name)
for action_type in engine.data.supported_action_types:
if action_type.name == action.data.action_type_name:
supported_action_type_params_schema = action_type.params_json_schema
break
return {'results': action_schema(supported_action_type_params_schema)}
def get_action_json_schema_empty():
supported_action_type_params_schema = None
action_id = request.args.get('action_id')
if action_id:
action = action_service().get_action(action_id, raise_when_missing=False)
if not action:
return {'results': 'ERROR', 'error_message': 'no action with id: %s' % action_id}, 404, None
engine_name = action.data.engine_name
engine = engine_service().get_engine_by_name(engine_name)
for action_type in engine.data.supported_action_types:
if action_type.name == action.data.action_type_name:
supported_action_type_params_schema = action_type.params_json_schema
break
return {'results': action_schema(supported_action_type_params_schema)}
def _query_action_query(self, filters, order_by=None):
""" :type filters: list[dart.model.query.Filter]
:type order_by: list[dart.model.query.OrderBy] """
action_types = []
for engine in self._engine_service.query_engines([], 1000, 0):
action_types.extend(engine.data.supported_action_types)
action_schemas = [action_schema(a.params_json_schema) for a in action_types]
query = ActionDao.query
for o in (order_by or []):
query = self._order_by_service.apply_order_by(o, query, ActionDao, action_schemas)
for f in filters:
query = self._filter_service.apply_filter(f, query, ActionDao, action_schemas)
return query
def test_action_schema(self):
last_in_workflow = None
a = Action(data=ActionData(
'copy_hdfs_to_s3',
'copy_hdfs_to_s3', {
'source_hdfs_path': 'hdfs:///user/hive/warehouse/dtest4',
'destination_s3_path': 's3://fake-bucket/dart_testing',
},
engine_name='no_op_engine',
last_in_workflow=last_in_workflow))
obj_before = a.to_dict()
obj_after = default_and_validate(
a,
action_schema(NoOpActionTypes.copy_hdfs_to_s3_action.
params_json_schema)).to_dict()
# many fields should have been defaulted, making these unequal
self.assertNotEqual(obj_before, obj_after)
def _query_action_query(self, filters, order_by=None):
""" :type filters: list[dart.model.query.Filter]
:type order_by: list[dart.model.query.OrderBy] """
action_types = []
for engine in self._engine_service.query_engines([], 1000, 0):
action_types.extend(engine.data.supported_action_types)
action_schemas = [action_schema(a.params_json_schema) for a in action_types]
query = ActionDao.query
for o in (order_by or []):
query = self._order_by_service.apply_order_by(o, query, ActionDao, action_schemas)
for f in filters:
query = self._filter_service.apply_filter(f, query, ActionDao, action_schemas)
return query
def test_action_schema_invalid(self):
with self.assertRaises(DartValidationException) as context:
a = Action(data=ActionData(
'copy_hdfs_to_s3',
'copy_hdfs_to_s3',
{
'source_hdfs_path': 'hdfs:///user/hive/warehouse/dtest4',
# 'destination_s3_path': 's3://fake-bucket/dart_testing',
},
engine_name='no_op_engine'))
# should fail because destination_s3_path is required
default_and_validate(
a,
action_schema(NoOpActionTypes.copy_hdfs_to_s3_action.
params_json_schema)).to_dict()
self.assertTrue(isinstance(context.exception, DartValidationException))
def save_subgraph_definition(subgraph_definition, engine, trigger_schemas):
""" :type engine: dart.model.engine.Engine
:type subgraph_definition: dart.model.graph.SubGraphDefinition """
action_schemas = [
action_schema(e.params_json_schema)
for e in engine.data.supported_action_types
]
ds_schema = datastore_schema(engine.data.options_json_schema)
schema = subgraph_definition_schema(trigger_schemas, action_schemas,
ds_schema)
subgraph_definition = default_and_validate(subgraph_definition, schema)
subgraph_definition_dao = SubGraphDefinitionDao()
subgraph_definition_dao.id = random_id()
subgraph_definition_dao.data = subgraph_definition.data.to_dict()
subgraph_definition_dao.data['engine_name'] = engine.data.name
db.session.add(subgraph_definition_dao)
db.session.commit()
return subgraph_definition_dao.to_model()
def main():
data = {
'definitions': {
'Action': action_schema(None),
'ActionContext': action_context_schema(),
'ActionResult': action_result_schema(),
'Dataset': dataset_schema(),
'Datastore': datastore_schema(None),
'Engine': engine_schema(),
'ErrorResult': error_result_schema(),
'Event': event_schema(),
'Filter': filter_schema(),
'GraphEntityIdentifier': graph_entity_identifier_schema(),
'GraphEntity': graph_entity_schema(),
'JSONPatch': json_patch_schema(),
'JSONSchema': json_schema_schema(),
'OKResult': ok_result_schema(),
'OrderBy': order_by_schema(),
'SubGraph': sub_graph_schema(),
'SubGraphDefinition': {
'type': 'object'
}, #subgraph_definition_schema([{'type': 'object'}], [{'type': 'object'}], {'type': 'object'}),
'Subscription': subscription_schema(),
'Trigger': trigger_schema({'type': 'object'}),
'TriggerType': trigger_type_schema(),
'Workflow': workflow_schema(),
'WorkflowInstance': workflow_instance_schema()
}
}
fix_up(data, data, [None])
print dump(data,
Dumper=Dumper,
default_style=None,
default_flow_style=False,
explicit_start=False,
explicit_end=False)
return 0
def export_swagger_definitions(out):
data = {
'definitions': {
'Action': action_schema({'type': 'object', 'x-nullable': True}),
'ActionResponse': object_response_schema('Action'),
'ActionsResponse': array_response_schema('Action'),
'PagedActionsResponse': paged_response_schema('Action'),
'ActionContext': action_context_schema(),
'ActionContextResponse': object_response_schema('ActionContext'),
'ActionResult': action_result_schema(),
'Dataset': dataset_schema(),
'DatasetResponse': object_response_schema('Dataset'),
'PagedDatasetsResponse': paged_response_schema('Dataset'),
'Datastore': datastore_schema({'type': 'object', 'x-nullable': True}),
'DatastoreResponse': object_response_schema('Datastore'),
'PagedDatastoresResponse': paged_response_schema('Datastore'),
'Engine': engine_schema(),
'EngineResponse': object_response_schema('Engine'),
'PagedEnginesResponse': paged_response_schema('Engine'),
'ErrorResponse': error_response_schema(),
'Event': event_schema(),
'EventResponse': object_response_schema('Event'),
'PagedEventsResponse': paged_response_schema('Event'),
'Filter': filter_schema(),
'GraphEntityIdentifier': graph_entity_identifier_schema(),
'GraphEntityIdentifierResponse': object_response_schema('GraphEntityIdentifier'),
'GraphEntityIdentifiersResponse': array_response_schema('GraphEntityIdentifier'),
'GraphEntity': graph_entity_schema(),
'GraphEntityResponse': object_response_schema('GraphEntity'),
'JSONPatch': json_patch_schema(),
'JSONSchema': json_schema_schema(),
'JSONSchemaResponse': object_response_schema('JSONSchema'),
'ObjectResponse': object_response_schema('object'),
'ObjectsResponse': array_response_schema('object'),
'PagedObjectsResponse': paged_response_schema('object'),
'OKResponse': ok_response_schema(),
'OrderBy': order_by_schema(),
'Subgraph': sub_graph_schema(),
'SubgraphResponse': object_response_schema('Subgraph'),
'SubgraphDefinition': {'type': 'object'}, #subgraph_definition_schema([{'type': 'object'}], [{'type': 'object'}], {'type': 'object'}),
'SubgraphDefinitionResponse': object_response_schema('SubgraphDefinition'),
'Subscription': subscription_schema(),
'SubscriptionResponse': object_response_schema('Subscription'),
'PagedSubscriptionsResponse': paged_response_schema('Subscription'),
'SubscriptionElement': subscription_element_schema(),
'PagedSubscriptionElementsResponse': paged_response_schema('SubscriptionElement'),
'Trigger': trigger_schema({'type': 'object'}),
'TriggerResponse': object_response_schema('Trigger'),
'PagedTriggersResponse': paged_response_schema('Trigger'),
'TriggerType': trigger_type_schema(),
'PagedTriggerTypesResponse': paged_response_schema('TriggerType'),
'Workflow': workflow_schema(),
'WorkflowResponse': object_response_schema('Workflow'),
'PagedWorkflowsResponse': paged_response_schema('Workflow'),
'WorkflowInstance': workflow_instance_schema(),
'WorkflowInstanceResponse': object_response_schema('WorkflowInstance'),
'PagedWorkflowInstancesResponse': paged_response_schema('WorkflowInstance')
}
}
fix_up(data, data, [None])
dump(data, out, Dumper=Dumper, default_style=None, default_flow_style=False, explicit_start=False, explicit_end=False)
def export_swagger_definitions(out):
data = {
'definitions': {
'Action':
action_schema({
'type': 'object',
'x-nullable': True
}),
'ActionResponse':
object_response_schema('Action'),
'ActionsResponse':
array_response_schema('Action'),
'PagedActionsResponse':
paged_response_schema('Action'),
'ActionContext':
action_context_schema(),
'ActionContextResponse':
object_response_schema('ActionContext'),
'ActionResult':
action_result_schema(),
'Dataset':
dataset_schema(),
'DatasetResponse':
object_response_schema('Dataset'),
'PagedDatasetsResponse':
paged_response_schema('Dataset'),
'Datastore':
datastore_schema({
'type': 'object',
'x-nullable': True
}),
'DatastoreResponse':
object_response_schema('Datastore'),
'PagedDatastoresResponse':
paged_response_schema('Datastore'),
'Engine':
engine_schema(),
'EngineResponse':
object_response_schema('Engine'),
'PagedEnginesResponse':
paged_response_schema('Engine'),
'ErrorResponse':
error_response_schema(),
'Event':
event_schema(),
'EventResponse':
object_response_schema('Event'),
'PagedEventsResponse':
paged_response_schema('Event'),
'Filter':
filter_schema(),
'GraphEntityIdentifier':
graph_entity_identifier_schema(),
'GraphEntityIdentifierResponse':
object_response_schema('GraphEntityIdentifier'),
'GraphEntityIdentifiersResponse':
array_response_schema('GraphEntityIdentifier'),
'GraphEntity':
graph_entity_schema(),
'GraphEntityResponse':
object_response_schema('GraphEntity'),
'JSONPatch':
json_patch_schema(),
'JSONSchema':
json_schema_schema(),
'JSONSchemaResponse':
object_response_schema('JSONSchema'),
'ObjectResponse':
object_response_schema('object'),
'ObjectsResponse':
array_response_schema('object'),
'PagedObjectsResponse':
paged_response_schema('object'),
'OKResponse':
ok_response_schema(),
'OrderBy':
order_by_schema(),
'Subgraph':
sub_graph_schema(),
'SubgraphResponse':
object_response_schema('Subgraph'),
'SubgraphDefinition': {
'type': 'object'
}, #subgraph_definition_schema([{'type': 'object'}], [{'type': 'object'}], {'type': 'object'}),
'SubgraphDefinitionResponse':
object_response_schema('SubgraphDefinition'),
'Subscription':
subscription_schema(),
'SubscriptionResponse':
object_response_schema('Subscription'),
'PagedSubscriptionsResponse':
paged_response_schema('Subscription'),
'SubscriptionElement':
subscription_element_schema(),
'PagedSubscriptionElementsResponse':
paged_response_schema('SubscriptionElement'),
'Trigger':
trigger_schema({'type': 'object'}),
'TriggerResponse':
object_response_schema('Trigger'),
'PagedTriggersResponse':
paged_response_schema('Trigger'),
'TriggerType':
trigger_type_schema(),
'PagedTriggerTypesResponse':
paged_response_schema('TriggerType'),
'Workflow':
workflow_schema(),
'WorkflowResponse':
object_response_schema('Workflow'),
'PagedWorkflowsResponse':
paged_response_schema('Workflow'),
'WorkflowInstance':
workflow_instance_schema(),
'WorkflowInstanceResponse':
object_response_schema('WorkflowInstance'),
'PagedWorkflowInstancesResponse':
paged_response_schema('WorkflowInstance')
}
}
fix_up(data, data, [None])
dump(data,
out,
Dumper=Dumper,
default_style=None,
default_flow_style=False,
explicit_start=False,
explicit_end=False)
