def getNestedPipelineData(self, files, path, userPath, filename): # Get the top level pipeline configuration file data. pipeline = pipelineConfiguration.pipelineConfiguration() pipeline.getConfigurationData(filename) pipeline.path = filename.rstrip(str(pipeline.name + ".json"))[:-1] self.pipelineConfigurationData[pipeline.name] = pipeline # Store the name of the tier 1 pipeline (e.g. the pipeline selected on the command line). self.tiersByPipeline[pipeline.name] = 1 self.pipelinesByTier[1] = [pipeline.name] self.pipeline = pipeline.name # Now dig into the nested pipeline and build up the superpipeline structure. tier = 2 checkForNestedPipelines = True while checkForNestedPipelines: tierHasNestedPipeline = False for currentPipelineName in self.pipelinesByTier[tier - 1]: currentPipeline = self.pipelineConfigurationData[currentPipelineName] # Check for external parameter sets for this pipeline. externalFilename = str(currentPipeline.path + "/" + currentPipeline.name + "-parameter-sets.json") # Get the configuration file data. data = fileHandling.fileHandling.readConfigurationFile(externalFilename, False) externalParameterSets = parameterSets.parameterSets() # If the external parameter set file exists, copy the parameter sets to the current pipeline configuration # data and mark them as external. try: parameterSetData = data["parameter sets"] except: parameterSetData = None if parameterSetData: if externalParameterSets.checkParameterSets(parameterSetData, False, currentPipelineName, False): for parameterSet in externalParameterSets.sets.keys(): if parameterSet in currentPipeline.parameterSets.sets.keys(): externalParameterSets.errors.externalSetRepeat(currentPipelineName, parameterSet) externalParameterSets.sets[parameterSet].isExternal = True currentPipeline.parameterSets.sets[parameterSet] = externalParameterSets.sets[parameterSet] # If the pipeline contains a pipeline as a task, process the configuration file for that # pipeline and add to the pipelines in the current tier. if currentPipeline.hasPipelineAsTask: tierHasNestedPipeline = True for taskPointingToNestedPipeline, nestedPipeline in currentPipeline.requiredPipelines: pipeline = pipelineConfiguration.pipelineConfiguration() # Check that the requried pipeline is available (first check in the user defined configuration # file path if defined). isPipeline = False if nestedPipeline in files.userConfigurationFiles: configurationPath = str(userPath) externalFilename = str(configurationPath + "/" + nestedPipeline + "-parameters-sets.json") filename = str(configurationPath + "/" + nestedPipeline + ".json") isPipeline = pipeline.checkConfigurationFile(filename) # If there is a pipeline configuration file with the given name in the user defined configuration # file directory, use this file. Otherwise, default to the standard gkno configuration files. if not isPipeline: configurationPath = str(path) externalFilename = str(path + "/" + nestedPipeline + "-parameters-sets.json") filename = str(path + "/" + nestedPipeline + ".json") # Process the configuration file. pipeline.getConfigurationData(filename) pipeline.path = configurationPath # Get the configuration file data. externalData = fileHandling.fileHandling.readConfigurationFile(externalFilename, False) if externalData: print( "TEST HANDLE EXTERNAL PARAMETER SET FOR NESETD PIPELINE", externalFilename, externalData ) exit(0) # Construct the address of the task. pipeline.address = "" if currentPipeline.address == None else str(currentPipeline.address) + "." pipeline.address += str(taskPointingToNestedPipeline) self.pipelineConfigurationData[pipeline.address] = pipeline # Store the pipeline name with the tier it's on. if tier not in self.pipelinesByTier: self.pipelinesByTier[tier] = [] self.pipelinesByTier[tier].append(pipeline.address) self.tiersByPipeline[pipeline.address] = tier # Increment the tier. If no pipelines in the just processed tier had a nested pipeline, the # loop can end. tier += 1 if not tierHasNestedPipeline: checkForNestedPipelines = False
def __init__(self): # Define error handling. self.errors = errors.toolErrors() # Record the name of the tool. self.name = None # Record the tool ID. self.id = None # Define the arguments associated with the tool. self.arguments = {} # Define the order in which the argument should be written. self.argumentOrder = [] # Define the delimiter between the argument and the value on the # tool commmand line. This is usually a space. self.delimiter = ' ' # A description of the tool. self.description = None # The categories to which the tool belongs. self.categories = ['General'] # The tool executable, its path and any modifiers. self.executable = None self.modifier = '' self.path = None self.precommand = '' # Record if this tool is hidden in the help. self.isHidden = False # Some tools do not produce any outputs. If this is the case, the tool has to # be marked. self.noOutput = False # Store the tools that need to be compiled for this tool to be available. self.requiredCompiledTools = [] # Keep track of any R packages that are required by the tool. self.rPackages = [] # Store the URL for the tool. self.url = None # If the tool is untested, but available, the isExperimental flag can be set. This # will ensure that the tool is listed as experimental and urge caution in its use. self.isDevelopmental = False # The parameter set information for this pipeline. self.parameterSets = parameterSets.parameterSets() # It is sometimes desirable to allow all steps to be processed without termination. Other # times, if a problem is encountered, execution should terminate. Keep track of this. self.allowTermination = True # Store information about the tool for display on the web page. self.webPage = {} # As pipeline configuration files are processed, success will identify whether a problem was # encountered. self.success = True
def __init__(self, allowTermination = True): # Handle errors. self.errors = errors.pipelineErrors() # Store the name of the pipeline. self.name = None # Store the path to the configuration file. self.path = None # Store the id for this pipeline. self.id = None # The pipeline description. self.description = 'No description' # The categories the pipeline is included in for help messages. self.categories = [] # The parameter set information for this pipeline. self.parameterSets = parameterSets.parameterSets() # The tasks that the pipeline comprises. Also store all of the tasks and all of the tools # that are executed by the pipeline. self.pipelineTasks = {} self.allTasks = [] self.allTools = [] # The pipeline graph nodes that are shared between different tasks in the pipeline. self.sharedNodeAttributes = {} # Pipeline graph nodes that are kept as unique for a specific task, self.uniqueNodeAttributes = {} # The connections that need to be made between nodes and tasks. self.connections = [] # Store all of the valid top level pipeline arguments. self.longFormArguments = {} self.shortFormArguments = {} # If the pipeline contains tasks that are themselves pipelines. Store all of the pipelines # used in this pipeline. self.hasPipelineAsTask = False self.requiredPipelines = [] # If there is a request to import argument from a tool, store the name of the tool. This tool # will not be checked for validity, that is left to the methods that use the tool. self.importArgumentsFromTool = None # If the pipeline is nested within other pipelines, the nodes associated with this pipeline # have an address to locate them within the graph structure. For example, if the main pipeline # has a task 'run' which is a pipeline, all the nodes created for this pipeline are prepended # with 'run.' etc. Store this address. self.address = None # It is sometimes desirable to allow all steps to be processed without termination. Other # times, if a problem is encountered, execution should terminate. Keep track of this. self.allowTermination = allowTermination # Flag if this pipeline contains instructions for building tasks that generate multiple output # file nodes. self.generatesMultipleNodes = False # Pipelines can be marked as developmental. This will keep them out of help messages and not include # them in the web json. self.isDevelopment = False # As pipeline configuration files are processed, success will identify whether a problem was # encountered. self.success = True