def findPipelinesInCategories(self, path):
categories = {}
descriptions = {}
for filename in os.listdir(path):
if filename.endswith(".json"):
pipeline = pc.pipelineConfiguration(allowTermination = False)
success = pipeline.getConfigurationData(path + '/' + filename)
# Store the pipeline description if the pipeline is not developmental.
if success and not pipeline.isDevelopment:
descriptions[pipeline.name] = pipeline.description
# Loop over the categories and populate the data structure.
for category in pipeline.categories:
# Check that the category is allowed.
if category not in self.helpCategories: self.errors.invalidCategory(pipeline.name, category, self.helpCategories)
if category not in categories: categories[str(category)] = [str(pipeline.name)]
else: categories[str(category)].append(str(pipeline.name))
# Return the dictionary containing all the tools connected to each category.
return categories, descriptions
def listAllPipelines(self, path):
self.writeSimpleLine('========================', isIndent = False, noLeadingTabs = 0)
self.writeSimpleLine('all gkno pipelines', isIndent = True, noLeadingTabs = 0)
self.writeSimpleLine('========================', isIndent = False, noLeadingTabs = 0)
print(file = sys.stdout)
self.writeSimpleLine('Following is a list of all of the pipelines currently available:', isIndent = False, noLeadingTabs = 0)
print(file = sys.stdout)
# Get the names and descriptions for all the pipelines.
descriptions = {}
failedPipelines = []
for filename in os.listdir(path):
if filename.endswith(".json"):
pipeline = pc.pipelineConfiguration()
pipeline.allowTermination = False
success = pipeline.getConfigurationData(path + '/' + filename)
# If the pipeline configuration was successfully parsed, get and store the description if the pipeline
# is not developmental.
if success:
if not pipeline.isDevelopment: descriptions[pipeline.name] = pipeline.description
# If there are errors with the pipeline configuration file, mark it as having errors.
else: failedPipelines.append(pipeline.name)
# Write out all the pipelines with their descriptions.
length = len(max(descriptions.keys(), key = len)) + 3
for pipeline in sorted(descriptions): self.writeComplexLine([pipeline + ':', descriptions[pipeline]], [length, 0], noLeadingTabs = 1)
print(file = sys.stdout)
# Now list all of the pipelines for which there exists a configuration file, but it has errors.
if failedPipelines:
self.writeSimpleLine('Following is a list of pipelines with malformed configuration files. These cannot be executed without first ' + \
'fixing the files. The pipeilne can be run to give a detailed description of the errors:', isIndent = False, noLeadingTabs = 0)
print(file = sys.stdout)
length = len(max(failedPipelines, key = len)) + 3
for pipeline in sorted(failedPipelines): self.writeComplexLine([pipeline], [length], noLeadingTabs = 1)
def getNestedPipelineData(self, files, path, userPath, filename):
# Get the top level pipeline configuration file data.
pipeline = pipelineConfiguration.pipelineConfiguration()
pipeline.getConfigurationData(filename)
pipeline.path = filename.rstrip(str(pipeline.name + ".json"))[:-1]
self.pipelineConfigurationData[pipeline.name] = pipeline
# Store the name of the tier 1 pipeline (e.g. the pipeline selected on the command line).
self.tiersByPipeline[pipeline.name] = 1
self.pipelinesByTier[1] = [pipeline.name]
self.pipeline = pipeline.name
# Now dig into the nested pipeline and build up the superpipeline structure.
tier = 2
checkForNestedPipelines = True
while checkForNestedPipelines:
tierHasNestedPipeline = False
for currentPipelineName in self.pipelinesByTier[tier - 1]:
currentPipeline = self.pipelineConfigurationData[currentPipelineName]
# Check for external parameter sets for this pipeline.
externalFilename = str(currentPipeline.path + "/" + currentPipeline.name + "-parameter-sets.json")
# Get the configuration file data.
data = fileHandling.fileHandling.readConfigurationFile(externalFilename, False)
externalParameterSets = parameterSets.parameterSets()
# If the external parameter set file exists, copy the parameter sets to the current pipeline configuration
# data and mark them as external.
try:
parameterSetData = data["parameter sets"]
except:
parameterSetData = None
if parameterSetData:
if externalParameterSets.checkParameterSets(parameterSetData, False, currentPipelineName, False):
for parameterSet in externalParameterSets.sets.keys():
if parameterSet in currentPipeline.parameterSets.sets.keys():
externalParameterSets.errors.externalSetRepeat(currentPipelineName, parameterSet)
externalParameterSets.sets[parameterSet].isExternal = True
currentPipeline.parameterSets.sets[parameterSet] = externalParameterSets.sets[parameterSet]
# If the pipeline contains a pipeline as a task, process the configuration file for that
# pipeline and add to the pipelines in the current tier.
if currentPipeline.hasPipelineAsTask:
tierHasNestedPipeline = True
for taskPointingToNestedPipeline, nestedPipeline in currentPipeline.requiredPipelines:
pipeline = pipelineConfiguration.pipelineConfiguration()
# Check that the requried pipeline is available (first check in the user defined configuration
# file path if defined).
isPipeline = False
if nestedPipeline in files.userConfigurationFiles:
configurationPath = str(userPath)
externalFilename = str(configurationPath + "/" + nestedPipeline + "-parameters-sets.json")
filename = str(configurationPath + "/" + nestedPipeline + ".json")
isPipeline = pipeline.checkConfigurationFile(filename)
# If there is a pipeline configuration file with the given name in the user defined configuration
# file directory, use this file. Otherwise, default to the standard gkno configuration files.
if not isPipeline:
configurationPath = str(path)
externalFilename = str(path + "/" + nestedPipeline + "-parameters-sets.json")
filename = str(path + "/" + nestedPipeline + ".json")
# Process the configuration file.
pipeline.getConfigurationData(filename)
pipeline.path = configurationPath
# Get the configuration file data.
externalData = fileHandling.fileHandling.readConfigurationFile(externalFilename, False)
if externalData:
print(
"TEST HANDLE EXTERNAL PARAMETER SET FOR NESETD PIPELINE", externalFilename, externalData
)
exit(0)
# Construct the address of the task.
pipeline.address = "" if currentPipeline.address == None else str(currentPipeline.address) + "."
pipeline.address += str(taskPointingToNestedPipeline)
self.pipelineConfigurationData[pipeline.address] = pipeline
# Store the pipeline name with the tier it's on.
if tier not in self.pipelinesByTier:
self.pipelinesByTier[tier] = []
self.pipelinesByTier[tier].append(pipeline.address)
self.tiersByPipeline[pipeline.address] = tier
# Increment the tier. If no pipelines in the just processed tier had a nested pipeline, the
# loop can end.
tier += 1
if not tierHasNestedPipeline:
checkForNestedPipelines = False
