def new(self, schema=workflow_metadata_template): """ Construct new empty workflow based on template JSON Schema file :param schema: JSON schema """ # Build workflow template from schema. # Set 'directed' to True to import JSON schema as directed graph template_graph = GraphAxis() template_graph.directed = True template = read_json_schema( schema, graph=template_graph, exclude_args=['title', 'description', 'schema_label']) self.workflow = template.query_nodes( key='project_metadata').descendants(include_self=True).copy() self.workflow.node_tools = NodeAxisTools self.workflow.orm = WORKFLOW_ORM renumber_id(self.workflow, 1) # Update workflow meta-data metadata = self.workflow.query_nodes(key='project_metadata') metadata.create_time.set() metadata.user.set() metadata.version.set(metadata.data.value_tag, __version__) logging.info('Init default empty workflow')
def load_task_schema(schema_name): """ Load task template graphs from JSON Schema template files Template files are located in the package /schemas/endpoints directory :param schema_name: task JSON Schema file name to load :type schema_name: :py:str :return: directed GraphAxis task template graph :rtype: :graphit:GraphAxis :raises: ImportError, unable to import JSON template """ # Set 'directed' to True to import JSON schema as directed graph template_graph = GraphAxis(directed=True) task_schema = pkg_resources.resource_filename( 'mdstudio_workflow', '/schemas/endpoints/{0}'.format(schema_name)) task_template = read_json_schema( task_schema, graph=template_graph, exclude_args=['title', 'description', 'schema_label']) task_node = task_template.query_nodes(key='task') if task_node.empty(): raise ImportError( 'Unable to load {0} task definitions'.format(schema_name)) return task_node
def setUp(self): """ Load a task specification from JSON Schema file """ self.task_graph = read_json_schema(self.task_schema, exclude_args=['description']) self.task_graph.orm = WORKFLOW_ORM self.task = self.task_graph.get_root()
def test_jsonschema_import(self): """ Test import of JSON schema """ schema = os.path.join( FILEPATH, 'jsonschema1.json', ) graph = read_json_schema(schema) # Test "enum" validation keyword node = graph.query_nodes(key='fstype') self.assertRaises(GraphitValidationError, node.set, 'value', 'ff')
def get_input(self, **kwargs): """ Prepare input dictionary for the WAMP call All data is transferred over the network by default in WAMP communication and therefor all file paths are read to string. """ input_dict = super(WampTask, self).get_input() # Retrieve JSON schemas for the endpoint request and response schemaparser = SchemaParser(kwargs.get('task_runner')) request = yield schemaparser.get(uri=self.uri(), request=True) request = read_json_schema(request) request.orm.node_mapping.add(FileType, lambda x: x.get('format') == 'file') request.orm.node_mapping.add(FileArrayType, lambda x: x.get('format') == 'file_array') # Register parameters wherever they are defined for key, value in input_dict.items(): node = request.query_nodes({request.data.key_tag: key}) if len(node) == 1: node.set(request.data.value_tag, value) elif node.empty(): logging.warn('Task task {0} ({1}): parameter {2} not defined in endpoint schema'.format(self.nid, self.key, key)) elif len(node) > 1: logging.warn('Task task {0} ({1}): parameter {2} defined multiple times in schema'.format(self.nid, self.key, key)) # Check for file types, remove schema parameters not defined for node in request.query_nodes({'format': u'file'}).iternodes(): if node.get(node.data.key_tag) not in input_dict: request.remove_node(node.nid) else: fileobj = node.to_dict(input_dict.get(node.key), inline_files=self.inline_files(), workdir=self.task_metadata.workdir.get()) for key, value in fileobj.items(): obj = node.descendants().query_nodes({node.data.key_tag: key}) obj.set(node.data.value_tag, value) # Reset original 'value' with file path node.set(node.data.value_tag, None) # Check for file arrays, remove schema parameters not defined for node in request.query_nodes({'format': u'file_array'}).iternodes(): if node.get(node.data.key_tag) not in input_dict: request.remove_node(node.nid) else: fileobj = node.to_dict(input_dict.get(node.key), inline_files=self.inline_files(), workdir=self.task_metadata.workdir.get()) # Reset original 'value' with file path node.set(node.data.value_tag, fileobj) # Check for parameters that have defaults, remove others nodes_to_remove = [] for node in request.query_nodes({u'schema_label': u'properties'}): if node.get(u'value') is None: node_type = node.get(u'type', []) if not isinstance(node_type, list): node_type = [node_type] if u'object' in node_type: continue if node.get(u'default') is None and u'null' not in node_type: nodes_to_remove.append(node.nid) request.remove_nodes(nodes_to_remove) # Build parameter dictionary from JSON Schema param_dict = write_pydata(request) # Remove all 'value' parameters with value None. # TODO: These are left overs from graphit. def recursive_remove_none(d): for k, v in d.items(): if k == 'value' and v is None: del d[k] elif isinstance(v, dict): d[k] = recursive_remove_none(v) return d param_dict = recursive_remove_none(param_dict) # Write input to disk as JSON? Task working directory should exist if self.task_metadata.store_output(): input_json = os.path.join(self.task_metadata.workdir.get(), 'input.json') json.dump(param_dict, open(input_json, 'w'), indent=2) return_value(param_dict)
import shutil import time from graphit.graph_io.io_jsonschema_format import read_json_schema from graphit.graph_io.io_web_format import read_web from graphit.graph_axis.graph_axis_mixin import NodeAxisTools from mdstudio_haddock.haddock_xmlrpc_endpoints import HaddockXmlrpcInterface from mdstudio_haddock.haddock_helper_methods import resolve_credentials from module.unittest_baseclass import UnittestPythonCompatibility # Parse package configuration file settings_file = pkg_resources.resource_filename('mdstudio_haddock', '/schemas/settings.json') xmlrpc_settings = read_json_schema(settings_file) xmlrpc_settings.node_tools = NodeAxisTools settings = xmlrpc_settings.getnodes(xmlrpc_settings.root).settings server_username, server_password = resolve_credentials(settings) @unittest.skipIf( not server_username or not server_password, "HADDOCK_SERVER_USER and HADDOCK_SERVER_PW environment variables not set.") class TestHaddockServerCommunication(UnittestPythonCompatibility): filedir = os.path.abspath( os.path.join(os.path.dirname(__file__), '../files/')) tempfiles = []