def getNestedPipelineData(self, files, path, userPath, filename):
# Get the top level pipeline configuration file data.
pipeline = pipelineConfiguration.pipelineConfiguration()
pipeline.getConfigurationData(filename)
pipeline.path = filename.rstrip(str(pipeline.name + ".json"))[:-1]
self.pipelineConfigurationData[pipeline.name] = pipeline
# Store the name of the tier 1 pipeline (e.g. the pipeline selected on the command line).
self.tiersByPipeline[pipeline.name] = 1
self.pipelinesByTier[1] = [pipeline.name]
self.pipeline = pipeline.name
# Now dig into the nested pipeline and build up the superpipeline structure.
tier = 2
checkForNestedPipelines = True
while checkForNestedPipelines:
tierHasNestedPipeline = False
for currentPipelineName in self.pipelinesByTier[tier - 1]:
currentPipeline = self.pipelineConfigurationData[currentPipelineName]
# Check for external parameter sets for this pipeline.
externalFilename = str(currentPipeline.path + "/" + currentPipeline.name + "-parameter-sets.json")
# Get the configuration file data.
data = fileHandling.fileHandling.readConfigurationFile(externalFilename, False)
externalParameterSets = parameterSets.parameterSets()
# If the external parameter set file exists, copy the parameter sets to the current pipeline configuration
# data and mark them as external.
try:
parameterSetData = data["parameter sets"]
except:
parameterSetData = None
if parameterSetData:
if externalParameterSets.checkParameterSets(parameterSetData, False, currentPipelineName, False):
for parameterSet in externalParameterSets.sets.keys():
if parameterSet in currentPipeline.parameterSets.sets.keys():
externalParameterSets.errors.externalSetRepeat(currentPipelineName, parameterSet)
externalParameterSets.sets[parameterSet].isExternal = True
currentPipeline.parameterSets.sets[parameterSet] = externalParameterSets.sets[parameterSet]
# If the pipeline contains a pipeline as a task, process the configuration file for that
# pipeline and add to the pipelines in the current tier.
if currentPipeline.hasPipelineAsTask:
tierHasNestedPipeline = True
for taskPointingToNestedPipeline, nestedPipeline in currentPipeline.requiredPipelines:
pipeline = pipelineConfiguration.pipelineConfiguration()
# Check that the requried pipeline is available (first check in the user defined configuration
# file path if defined).
isPipeline = False
if nestedPipeline in files.userConfigurationFiles:
configurationPath = str(userPath)
externalFilename = str(configurationPath + "/" + nestedPipeline + "-parameters-sets.json")
filename = str(configurationPath + "/" + nestedPipeline + ".json")
isPipeline = pipeline.checkConfigurationFile(filename)
# If there is a pipeline configuration file with the given name in the user defined configuration
# file directory, use this file. Otherwise, default to the standard gkno configuration files.
if not isPipeline:
configurationPath = str(path)
externalFilename = str(path + "/" + nestedPipeline + "-parameters-sets.json")
filename = str(path + "/" + nestedPipeline + ".json")
# Process the configuration file.
pipeline.getConfigurationData(filename)
pipeline.path = configurationPath
# Get the configuration file data.
externalData = fileHandling.fileHandling.readConfigurationFile(externalFilename, False)
if externalData:
print(
"TEST HANDLE EXTERNAL PARAMETER SET FOR NESETD PIPELINE", externalFilename, externalData
)
exit(0)
# Construct the address of the task.
pipeline.address = "" if currentPipeline.address == None else str(currentPipeline.address) + "."
pipeline.address += str(taskPointingToNestedPipeline)
self.pipelineConfigurationData[pipeline.address] = pipeline
# Store the pipeline name with the tier it's on.
if tier not in self.pipelinesByTier:
self.pipelinesByTier[tier] = []
self.pipelinesByTier[tier].append(pipeline.address)
self.tiersByPipeline[pipeline.address] = tier
# Increment the tier. If no pipelines in the just processed tier had a nested pipeline, the
# loop can end.
tier += 1
if not tierHasNestedPipeline:
checkForNestedPipelines = False
def __init__(self):
# Define error handling.
self.errors = errors.toolErrors()
# Record the name of the tool.
self.name = None
# Record the tool ID.
self.id = None
# Define the arguments associated with the tool.
self.arguments = {}
# Define the order in which the argument should be written.
self.argumentOrder = []
# Define the delimiter between the argument and the value on the
# tool commmand line. This is usually a space.
self.delimiter = ' '
# A description of the tool.
self.description = None
# The categories to which the tool belongs.
self.categories = ['General']
# The tool executable, its path and any modifiers.
self.executable = None
self.modifier = ''
self.path = None
self.precommand = ''
# Record if this tool is hidden in the help.
self.isHidden = False
# Some tools do not produce any outputs. If this is the case, the tool has to
# be marked.
self.noOutput = False
# Store the tools that need to be compiled for this tool to be available.
self.requiredCompiledTools = []
# Keep track of any R packages that are required by the tool.
self.rPackages = []
# Store the URL for the tool.
self.url = None
# If the tool is untested, but available, the isExperimental flag can be set. This
# will ensure that the tool is listed as experimental and urge caution in its use.
self.isDevelopmental = False
# The parameter set information for this pipeline.
self.parameterSets = parameterSets.parameterSets()
# It is sometimes desirable to allow all steps to be processed without termination. Other
# times, if a problem is encountered, execution should terminate. Keep track of this.
self.allowTermination = True
# Store information about the tool for display on the web page.
self.webPage = {}
# As pipeline configuration files are processed, success will identify whether a problem was
# encountered.
self.success = True
def __init__(self, allowTermination = True):
# Handle errors.
self.errors = errors.pipelineErrors()
# Store the name of the pipeline.
self.name = None
# Store the path to the configuration file.
self.path = None
# Store the id for this pipeline.
self.id = None
# The pipeline description.
self.description = 'No description'
# The categories the pipeline is included in for help messages.
self.categories = []
# The parameter set information for this pipeline.
self.parameterSets = parameterSets.parameterSets()
# The tasks that the pipeline comprises. Also store all of the tasks and all of the tools
# that are executed by the pipeline.
self.pipelineTasks = {}
self.allTasks = []
self.allTools = []
# The pipeline graph nodes that are shared between different tasks in the pipeline.
self.sharedNodeAttributes = {}
# Pipeline graph nodes that are kept as unique for a specific task,
self.uniqueNodeAttributes = {}
# The connections that need to be made between nodes and tasks.
self.connections = []
# Store all of the valid top level pipeline arguments.
self.longFormArguments = {}
self.shortFormArguments = {}
# If the pipeline contains tasks that are themselves pipelines. Store all of the pipelines
# used in this pipeline.
self.hasPipelineAsTask = False
self.requiredPipelines = []
# If there is a request to import argument from a tool, store the name of the tool. This tool
# will not be checked for validity, that is left to the methods that use the tool.
self.importArgumentsFromTool = None
# If the pipeline is nested within other pipelines, the nodes associated with this pipeline
# have an address to locate them within the graph structure. For example, if the main pipeline
# has a task 'run' which is a pipeline, all the nodes created for this pipeline are prepended
# with 'run.' etc. Store this address.
self.address = None
# It is sometimes desirable to allow all steps to be processed without termination. Other
# times, if a problem is encountered, execution should terminate. Keep track of this.
self.allowTermination = allowTermination
# Flag if this pipeline contains instructions for building tasks that generate multiple output
# file nodes.
self.generatesMultipleNodes = False
# Pipelines can be marked as developmental. This will keep them out of help messages and not include
# them in the web json.
self.isDevelopment = False
# As pipeline configuration files are processed, success will identify whether a problem was
# encountered.
self.success = True
