def new(self, schema=workflow_metadata_template):
"""
Construct new empty workflow based on template JSON Schema file
:param schema: JSON schema
"""
# Build workflow template from schema.
# Set 'directed' to True to import JSON schema as directed graph
template_graph = GraphAxis()
template_graph.directed = True
template = read_json_schema(
schema,
graph=template_graph,
exclude_args=['title', 'description', 'schema_label'])
self.workflow = template.query_nodes(
key='project_metadata').descendants(include_self=True).copy()
self.workflow.node_tools = NodeAxisTools
self.workflow.orm = WORKFLOW_ORM
renumber_id(self.workflow, 1)
# Update workflow meta-data
metadata = self.workflow.query_nodes(key='project_metadata')
metadata.create_time.set()
metadata.user.set()
metadata.version.set(metadata.data.value_tag, __version__)
logging.info('Init default empty workflow')
def load_task_schema(schema_name):
"""
Load task template graphs from JSON Schema template files
Template files are located in the package /schemas/endpoints directory
:param schema_name: task JSON Schema file name to load
:type schema_name: :py:str
:return: directed GraphAxis task template graph
:rtype: :graphit:GraphAxis
:raises: ImportError, unable to import JSON template
"""
# Set 'directed' to True to import JSON schema as directed graph
template_graph = GraphAxis(directed=True)
task_schema = pkg_resources.resource_filename(
'mdstudio_workflow', '/schemas/endpoints/{0}'.format(schema_name))
task_template = read_json_schema(
task_schema,
graph=template_graph,
exclude_args=['title', 'description', 'schema_label'])
task_node = task_template.query_nodes(key='task')
if task_node.empty():
raise ImportError(
'Unable to load {0} task definitions'.format(schema_name))
return task_node
def setUp(self):
"""
Load a task specification from JSON Schema file
"""
self.task_graph = read_json_schema(self.task_schema,
exclude_args=['description'])
self.task_graph.orm = WORKFLOW_ORM
self.task = self.task_graph.get_root()
def test_jsonschema_import(self):
"""
Test import of JSON schema
"""
schema = os.path.join(
FILEPATH,
'jsonschema1.json',
)
graph = read_json_schema(schema)
# Test "enum" validation keyword
node = graph.query_nodes(key='fstype')
self.assertRaises(GraphitValidationError, node.set, 'value', 'ff')
def get_input(self, **kwargs):
"""
Prepare input dictionary for the WAMP call
All data is transferred over the network by default in WAMP
communication and therefor all file paths are read to string.
"""
input_dict = super(WampTask, self).get_input()
# Retrieve JSON schemas for the endpoint request and response
schemaparser = SchemaParser(kwargs.get('task_runner'))
request = yield schemaparser.get(uri=self.uri(), request=True)
request = read_json_schema(request)
request.orm.node_mapping.add(FileType, lambda x: x.get('format') == 'file')
request.orm.node_mapping.add(FileArrayType, lambda x: x.get('format') == 'file_array')
# Register parameters wherever they are defined
for key, value in input_dict.items():
node = request.query_nodes({request.data.key_tag: key})
if len(node) == 1:
node.set(request.data.value_tag, value)
elif node.empty():
logging.warn('Task task {0} ({1}): parameter {2} not defined in endpoint schema'.format(self.nid,
self.key, key))
elif len(node) > 1:
logging.warn('Task task {0} ({1}): parameter {2} defined multiple times in schema'.format(self.nid,
self.key, key))
# Check for file types, remove schema parameters not defined
for node in request.query_nodes({'format': u'file'}).iternodes():
if node.get(node.data.key_tag) not in input_dict:
request.remove_node(node.nid)
else:
fileobj = node.to_dict(input_dict.get(node.key),
inline_files=self.inline_files(),
workdir=self.task_metadata.workdir.get())
for key, value in fileobj.items():
obj = node.descendants().query_nodes({node.data.key_tag: key})
obj.set(node.data.value_tag, value)
# Reset original 'value' with file path
node.set(node.data.value_tag, None)
# Check for file arrays, remove schema parameters not defined
for node in request.query_nodes({'format': u'file_array'}).iternodes():
if node.get(node.data.key_tag) not in input_dict:
request.remove_node(node.nid)
else:
fileobj = node.to_dict(input_dict.get(node.key),
inline_files=self.inline_files(),
workdir=self.task_metadata.workdir.get())
# Reset original 'value' with file path
node.set(node.data.value_tag, fileobj)
# Check for parameters that have defaults, remove others
nodes_to_remove = []
for node in request.query_nodes({u'schema_label': u'properties'}):
if node.get(u'value') is None:
node_type = node.get(u'type', [])
if not isinstance(node_type, list):
node_type = [node_type]
if u'object' in node_type:
continue
if node.get(u'default') is None and u'null' not in node_type:
nodes_to_remove.append(node.nid)
request.remove_nodes(nodes_to_remove)
# Build parameter dictionary from JSON Schema
param_dict = write_pydata(request)
# Remove all 'value' parameters with value None.
# TODO: These are left overs from graphit.
def recursive_remove_none(d):
for k, v in d.items():
if k == 'value' and v is None:
del d[k]
elif isinstance(v, dict):
d[k] = recursive_remove_none(v)
return d
param_dict = recursive_remove_none(param_dict)
# Write input to disk as JSON? Task working directory should exist
if self.task_metadata.store_output():
input_json = os.path.join(self.task_metadata.workdir.get(), 'input.json')
json.dump(param_dict, open(input_json, 'w'), indent=2)
return_value(param_dict)
import shutil
import time
from graphit.graph_io.io_jsonschema_format import read_json_schema
from graphit.graph_io.io_web_format import read_web
from graphit.graph_axis.graph_axis_mixin import NodeAxisTools
from mdstudio_haddock.haddock_xmlrpc_endpoints import HaddockXmlrpcInterface
from mdstudio_haddock.haddock_helper_methods import resolve_credentials
from module.unittest_baseclass import UnittestPythonCompatibility
# Parse package configuration file
settings_file = pkg_resources.resource_filename('mdstudio_haddock',
'/schemas/settings.json')
xmlrpc_settings = read_json_schema(settings_file)
xmlrpc_settings.node_tools = NodeAxisTools
settings = xmlrpc_settings.getnodes(xmlrpc_settings.root).settings
server_username, server_password = resolve_credentials(settings)
@unittest.skipIf(
not server_username or not server_password,
"HADDOCK_SERVER_USER and HADDOCK_SERVER_PW environment variables not set.")
class TestHaddockServerCommunication(UnittestPythonCompatibility):
filedir = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../files/'))
tempfiles = []