def convertPipeArgumentsToToolArguments(self, arguments, shortForms, pipelineArguments, verbose): er = errors() shortForm = '' value = '' for argument in self.instanceData: if argument == 'description': self.description = self.instanceData[argument] else: if argument in arguments: shortForm = arguments[argument]['short form argument'] value = self.instanceData[argument] elif argument in shortForms: shortForm = argument argument = shortForms[argument] value = self.instanceData[shortForm] else: er.invalidArgumentInInstance(verbose, self.instanceName, argument) er.terminate() # Determine the tool and argument to which the pipeline argument points. tool = arguments[argument]['link to this task'] if tool == 'pipeline': pipelineArguments[argument] = value else: toolArgument = arguments[argument]['link to this argument'] if tool not in self.arguments: self.arguments[tool] = {} if toolArgument not in self.arguments[tool]: self.arguments[tool][toolArgument] = [] if type(value) == list: for entry in value: self.arguments[tool][toolArgument].append(entry) else: self.arguments[tool][toolArgument].append(value)
def __init__(self): self.errors = errors() self.toolArgumentLinks = {} self.arguments = {} self.argumentInformation = {} self.constructFilenames = {} self.deleteFiles = {} self.description = '' self.instances = {} self.isPipeline = False self.linkage = {} self.pipelineName = '' self.shortForms = {} self.taskToTool = {} self.toolsOutputtingToStream = [] self.workflow = [] # Define the fields that are allowed for each argument in the linkage section of the # configuration file. The Boolean value describes if the field is required or not. self.allowedLinkageFields = {} self.allowedLinkageFields['link to this task'] = True self.allowedLinkageFields['link to this argument'] = True self.allowedLinkageFields['json block'] = False self.allowedLinkageFields['extension'] = False # Define the fields that are allowed for each argument in the construct filenames section # of the configuration file. The Boolean value describes if the field is required or not. self.allowedConstructFields = {} self.allowedConstructFields['filename root'] = True self.allowedConstructFields['filename root text'] = False self.allowedConstructFields['get root from task'] = False self.allowedConstructFields['get root from argument'] = False self.allowedConstructFields['remove input extension'] = False self.allowedConstructFields['additional text from parameters'] = False self.allowedConstructFields['output extension'] = False
def checkInstanceInformation(self, instances, storedInstances, filename): er = errors() if 'instances' not in instances: er.instancesFileHasNoInstances(False, filename) er.terminate() instances = instances['instances'] givenType = type(instances) if givenType != dict: er.differentDataTypeInConfig(False, filename, '', 'instances', givenType, dict) er.terminate() # Loop over the instances and add them to the storedInstances structure. for instance in instances: # Check that the instances in the instances file do not have the same name as any in the # configuration file. if instance in storedInstances: er.instanceNameAlreadyExists(False, instance, filename) er.terminate() storedInstances[instance] = instances[instance] # Also store the instance name in the externalInstances structure. This keeps track of # the instances in the instances file (and not in the configuration file), if the instances # file gets modified (using the export instance functionality). self.externalInstances.append(instance)
def determineAdditionalFiles(additionalFiles, workflow, taskToTool, arguments, dependencies, outputs, verbose): er = errors() # Loop over each tool in turn and check for additional output files. for task in workflow: if task not in outputs: outputs[task] = [] tool = taskToTool[task] if tool in additionalFiles: # There are different formats for building up output files. Each of these can # be dealt with separately. # # 1. If the output file can be constructed from a value given to one of the # tools command line arguments. if 'from input arguments' in additionalFiles[tool]: for field in additionalFiles[tool]['from input arguments']: # Check that the command which should be used to determine the file name exists # and has a value defined. fileType = field['type'] argument = field['link to this argument'] filename = arguments[task][argument][0] # In constructing the output file name, the extension associated with the associated # file name can be stripped off and a new extension can be appended if requested. # Determine and enact the appropriate steps. if field['remove extension']: filename = filename.rpartition('.')[0] if field['add extension']: extension = field['output extension'] filename += '.' + extension # If the file is a dependency, add to the dependency string, otherwise add to the # output string. if fileType == 'dependency': dependencies[task].append(filename) elif fileType == 'output': outputs[task].append(filename)
def __init__(self, sourcePath): # Command line modes self.resourceModes = [ "add-resource", "remove-resource", "update-resource" ] self.allModes = [ "build", "update" ] + self.resourceModes # Setup help/usage descriptions self.modeDescriptions = {} self.modeDescriptions["build"] = "Initialize gkno. This step is required to run any other operations." self.modeDescriptions["update"] = "Update gkno itself, its internal tools, and any tracked organism resources." self.modeDescriptions["add-resource"] = "Download resource data for an organism and track it for updated releases." self.modeDescriptions["remove-resource"] = "Delete resource data for an organism and stop tracking it." self.modeDescriptions["update-resource"] = "Update resource data for an organism." # General data members self.error = errors() self.isRequested = False self.isVerbose = False self.mode = "" self.userSettings = { } # Commonly used path names self.sourcePath = sourcePath self.resourcesPath = sourcePath + "/resources/" self.toolsPath = sourcePath + "/tools/" self.logsPath = sourcePath + "/logs/" # Import user settings from file (or initialize defaults if file doesn't exist) self.importUserSettings() # Make sure our common directories exist self.ensureMakeDir(self.resourcesPath) self.ensureMakeDir(self.toolsPath) self.ensureMakeDir(self.logsPath)
def __init__(self): self.arguments = {} self.argumentList = [] self.data = [] self.errors = errors() self.ID = 0 self.numberOfIterations = 0 self.tasks = [] self.usingInternalLoop = False
def __init__(self): # Get general error writing and termination methods. self.errors = errors() # The error messages are stored in the following list. self.text = [] # For a list of all error code values, see adminErrors.py. self.errorCode = '12'
def getInstanceArguments(self, path, name, instances, verbose): er = errors() if self.instanceName not in instances: externalInstancesFilename = name + '_instances.json' io = files() data = io.getJsonData(path + externalInstancesFilename, False) if data == '': er.noInstanceInformation(verbose, path, name, self.instanceName) er.terminate() self.externalInstance = True self.instanceData = data['instances'][self.instanceName] else: self.instanceData = instances[self.instanceName]
def checkExportName(self, tl, io): er = errors() self.outputName = tl.toolArguments['pipeline']['--export-config'] # Check that the filename ends with '.json'. if self.outputName[-5:len(self.outputName)] != '.json': self.outputName += '.json' # Check that the configuration file does not already exist. for json in io.jsonPipelineFiles: if self.outputName == json: er.outputJsonExists(True, "\t", io, self.outputName) er.terminate()
def getInformation(self, verbose): er = errors() # Attempt to open the json file containing the information. io = files() data = io.getJsonData(self.filename, False) if data == '': er.noMultipleRunFile(False, self.filename) er.terminate() # The multiple runs file should contain two sections: 'format of data list' and 'data list'. # Check that these exist and store the information if 'format of data list' not in data: er.missingSectionMultipleRunsFile(False, self.filename, 'format of data list') er.terminate() if 'data list' not in data: er.missingSectionMultipleRunsFile(False, self.filename, 'data list') er.terminate() # Check that the "format of data list" is well formed and store. givenType = type(data['format of data list']) if givenType != list: er.differentDataTypeInConfig(False, self.filename, '', 'format of data list', list, givenType) er.terminate() # Ensure that the long form of the argument is used in the argumentFormats structure. for argument in data['format of data list']: self.argumentFormats.append(argument) # Now check the data for the "data list" and store the information. givenType = type(data['data list']) if givenType != list: er.differentDataTypeInConfig(False, self.filename, '', 'data list', list, givenType) er.terminate() # Check that there is a valid number of entries in the list. For example, if the argumentFormats list # has two entries, the data list must contain two entries for each run (i.e. one value for each argument # defined in the formats for each run). Thus the number of entries in the 'data list' must be a # multiple of the number of entries in the 'format of data list' section. if len(data['data list']) % len(self.argumentFormats) != 0: er.incorrectNumberOfEntriesInMultipleJson(False, self.filename) er.terminate() self.numberDataSets = len(data['data list']) / len(self.argumentFormats) count = 0 for argument in data['data list']: if self.argumentFormats[count] not in self.argumentData: self.argumentData[self.argumentFormats[count]] = [] self.argumentData[self.argumentFormats[count]].append(argument) count += 1 if count == len(self.argumentFormats): count = 0
def checkInformation(self, singleTask, arguments, shortForms, verbose): er = errors() longFormFormats = [] longFormData = {} for argument in self.argumentData: values = self.argumentData[argument] # Check that the argument is valid. if argument in arguments: longForm = argument shortForm = arguments[argument]['short form argument'] elif argument in shortForms: longForm = shortForms[argument] shortForm = argument else: er.unknownArgumentInMultipleRuns(False, self.filename, argument) er.terminate() if 'link to this task' in arguments[longForm]: task = arguments[longForm]['link to this task'] link = arguments[longForm]['link to this argument'] else: task = singleTask link = longForm # Now check the data associated with this argument. dataType = arguments[longForm]['type'] for value in self.argumentData[argument]: success = checkDataType(dataType, value) if not success: if dataType == 'flag': er.flagGivenInvalidValueMultiple(verbose, self.filename, argument, shortForm, value) elif dataType == 'bool': er.incorrectBooleanValueInMultiple(verbose, self.filename, argument, shortForm, value) elif dataType == 'integer': er.incorrectDataTypeInMultiple(verbose, self.filename, argument, shortForm, value, dataType) elif dataType == 'float': er.incorrectDataTypeInMultiple(verbose, self.filename, argument, shortForm, value, dataType) elif dataType == 'string': er.incorrectDataTypeInMultiple(verbose, self.filename, argument, shortForm, value, dataType) else: print('error with data type 2') er.terminate() # Add the values to the data structures. longFormFormats.append(longForm) longFormData[longForm] = self.argumentData[argument] if task not in self.allArguments: self.allArguments[task] = {} self.allArguments[task][link] = self.argumentData[argument] # Replace the original argumentFormats structure with the longFormFormats to ensure that # the list only contains the long forms of the arguments. self.argumentFormats = longFormFormats self.argumentData = longFormData
def getExecutablePath(self, paths, taskToTool, taskBlock): er = errors() self.pathList = [] for task in taskBlock: tool = taskToTool[task] path = paths[tool] # Only print out the path if it isn't defined as 'null'. Unix commands will # be defined as 'null' as they do not need to include a path, for example. pathVariable = '' if path != 'no path': pathVariable = (tool.replace(" ", "_") + '_PATH').upper() print(pathVariable, "=$(TOOL_BIN)/", path, sep = "" , file = self.makeFilehandle) self.pathList.append(pathVariable)
def getInstanceName(self, uniqueArguments, argumentList, verbose): er = errors() # Check if the --instance (-is) command appears on the command line. if '--instance' in uniqueArguments: # If multiple instances have been requested, fail. if uniqueArguments['--instance'] != 1: er.multipleInstances(verbose) er.terminate() # Find the name of the requested instance and check that it is valid. for argument, instance in argumentList: if argument == '--instance': self.instanceName = instance break else: self.instanceName = 'default'
def checkInputLists(argumentInformation, workflow, taskToTool, arguments, verbose): er = errors() modifiedArguments = deepcopy(arguments) argumentsToRemove = [] for task in workflow: tool = taskToTool[task] if task not in modifiedArguments: modifiedArguments[task] = {} for argument in arguments[task]: isList = False if argument != 'json parameters': if 'list of input files' in argumentInformation[tool][argument]: if argumentInformation[tool][argument]['list of input files'] == True: isList = True if isList: repeatArgument = argumentInformation[tool][argument]['apply by repeating this argument'] # Check that the file exists, open and read the contents. The file should # be a simple json list. Only do this is a file is given. filename = arguments[task][argument][0] io = files() data = io.getJsonData(filename, False) if data == '': er.noInputFileList(verbose, filename) er.terminate() if 'filename list' not in data: er.hasError = True else: if not isinstance(data['filename list'], list): er.hasError = True if er.hasError: er.malformedFilenameList(verbose, filename, task, tool, argument) er.terminate() # All of the data has been read in and is valid, so can be added to the # lists of values for the specified argument. if repeatArgument not in modifiedArguments[task]: modifiedArguments[task][repeatArgument] = [] for inputFile in data['filename list']: modifiedArguments[task][repeatArgument].append(inputFile) # Having accounted for all the files in the list, this option can be removed # from the tools data structure. argumentsToRemove.append((task, argument)) for task, argument in argumentsToRemove: del modifiedArguments[task][argument] return modifiedArguments
def determineFilesToDelete(arguments, deleteFiles, verbose): er = errors() output = {} # Check to see if the configuration file has the 'delete files' section. if len(deleteFiles) != 0: for task in deleteFiles: for information in deleteFiles[task]: argument = information[0] deleteAfterTask = information[1] extension = information[2] if deleteAfterTask not in output: output[deleteAfterTask] = {} if argument not in output[deleteAfterTask]: output[deleteAfterTask][argument] = [] for value in arguments[task][argument]: output[deleteAfterTask][argument].append(value + extension) return output
def checkForMultipleRuns(self, uniqueArguments, argumentList, verbose): er = errors() # First, check if multiple runs have been requested. if '--multiple-runs' in uniqueArguments: self.hasMultipleRuns = True # Get the name of the file containing the required information for the multiple runs and check that # the file exists and is a valid json. for argument, value in argumentList: if argument == '--multiple-runs': self.filename = value break # If the --multiple-runs argument was not given a value on the command line, terminate. if self.filename == '': er.missingArgumentValue(verbose, '', 'pipeline', '--multiple-runs', '-mr', '', '', 'string') er.terminate()
def __init__(self): # Set logging verbose(self, self.__class__.__name__, loglevel=10, screen=True) self.vout.debug("Logging enabled.") self.pagedata = None self.err = errors(self) # Set browser session self.br = requests.Session() #Set session headers headers = {} headers['content-type'] = "application/json" headers['User-agent'] = ( 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1)' + ' Gecko/2008071615' + ' Fedora/3.0.1-1.fc9' + ' Firefox/3.0.1') self.br.headers.update(headers)
def __init__(self): self.errors = errors() # Set up a list of top level fields that must be present for all tools. self.requiredFields = [] self.requiredFields.append('description') self.requiredFields.append('help') self.requiredFields.append('path') self.requiredFields.append('executable') self.requiredFields.append('arguments') self.requiredFields.append('instances') # The configuration file contains arguments for each tool. Each argument also has a # set of required fields that must be checked. self.requiredArgumentFields = [] self.requiredArgumentFields.append('description') self.requiredArgumentFields.append('dependent') self.requiredArgumentFields.append('extension') self.requiredArgumentFields.append('input') self.requiredArgumentFields.append('output') self.requiredArgumentFields.append('required') self.requiredArgumentFields.append('type') # Setup some structures to hold information on each of the tools. self.additionalFiles = {} self.arguments = {} self.argumentInformation = {} self.argumentDelimiters = {} self.argumentOrder = {} self.availableTools = [] self.descriptions = {} self.executables = {} self.generatedFiles = {} self.instances = {} self.modifiers = {} self.paths = {} self.precommands = {} self.shortForms = {} self.hiddenTools = {} self.tool = '' self.toolsDemandingInputStream = {} self.toolsDemandingOutputStream = {}
def checkInstanceFile(self, argumentList, filename, instances, verbose): # , toolFiles, pipelineFiles, verbose): er = errors() for argument, value in argumentList: if argument == "--export-instance": self.name = value break # Set the filename. self.filename = filename + "_instances.json" # If no filename was supplied for the output configuration file, terminate. if self.name == "": er.noInstanceNameInExport(verbose, instanceName, self.filename) er.terminate() # Check that an instance of the given name doesn't already exist. for instance in instances: if self.name == instance: er.instanceNameExists(verbose, self.name) er.terminate()
def checkInstanceArguments(self, taskToTool, arguments, shortForms, verbose): er = errors() shortForm = '' value = '' modifiedArguments = {} # Clear the instanceData structure as it is no longer required. self.instanceData = {} for task in self.arguments: modifiedArguments[task] = {} tool = taskToTool[task] for argument in self.arguments[task]: for value in self.arguments[task][argument]: # Ensure that the argument is in the long form. if argument in arguments[tool]: shortForm = arguments[tool][argument]['short form argument'] if 'short form argument' in arguments[tool][argument] else '' elif argument in shortForms[tool]: shortForm = argument argument = shortForms[tool][argument] else: er.unknownArgumentInInstance(verbose, self.instanceName, argument) er.terminate() # Check the data type. dataType = arguments[tool][argument]['type'] success = checkDataType(dataType, value) if not success: if dataType == 'flag': er.flagGivenInvalidValueInstance(verbose, argument, shortForm, value) elif dataType == 'bool': er.incorrectBooleanValueInInstance(verbose, self.instanceName, argument, shortForm, value) elif dataType == 'integer': er.incorrectDataTypeInInstance(verbose, self.instanceName, task, argument, shortForm, value, dataType) elif dataType == 'float': er.incorrectDataTypeInInstance(verbose, self.instanceName, task, argument, shortForm, value, dataType) elif dataType == 'string': er.incorrectDataTypeInInstance(verbose, self.instanceName, task, argument, shortForm, value, dataType) else: print('error with data type 1', givenType) er.terminate() else: if argument not in modifiedArguments[task]: modifiedArguments[task][argument] = [] modifiedArguments[task][argument].append(value) # Update the arguments structrue to contain the modifiedArguments (where all arguments are in their long forms). self.arguments = modifiedArguments
def checkExtension(argumentInformation, shortForms, links, task, tool, argument, isOutput, filename, verbose): er = errors() correct = False # First check if the filename is a stub. isStub = argumentInformation[tool][argument]['stub'] if 'stub' in argumentInformation[tool][argument] else False # Only find the extension and check the files if the filename isn't a stub. if not isStub: # The extension could be a list of allowable extensions separated by pipes. # Split the extension on the pipes and check the files extension against # all the allowed extensions. extensions = argumentInformation[tool][argument]['extension'].split('|') # If the extension is listed as null, do not check the extension as it is # permitted to be anything. if (len(extensions) == 1) and (extensions[0] == 'no extension'): next else: for extension in extensions: extension = '.' + extension if filename.endswith(extension): correct = True break # If all the extensions have been checked and the files extension did not # match any of them, throw an error. if not correct and isOutput: filename += '.' + extensions[0] elif not correct: # Find the short form of the argument, as well as the pipeline version of the argument if one exists. shortForm = shortForms[argument] if argument in shortForms else '' pipelineLink = '' if task in links: if argument in links[task]: pipelineLink = links[task][argument] er.fileExtensionError(verbose, task, argument, shortForm, pipelineLink, filename, argumentInformation[tool][argument]['extension']) er.terminate() return filename
def getJsonData(self, filename, fail): er = errors() inputData = '' # Check that the file exists. try: jsonData = open(filename) except: er.hasError = True if er.hasError and fail: er.missingFile(True, filename) er.terminate() if not er.hasError: try: inputData = json.load(jsonData) except: er.hasError = True exc_type, exc_value, exc_traceback = sys.exc_info() # If the json file has a problem, terminate the script with an error. if er.hasError: er.jsonOpenError(True, exc_value, filename) er.terminate() return inputData
def constructFilenameFromInput(task, tool, argumentInformation, arguments, outputFile, inputArgument, verbose): er = errors() # Check if the input file that is to be used for constructing the output filename is # blank. If so, terminate gkno as the output filename cannot be determined. if len(arguments[task][inputArgument]) == 0: return # If the output filename is to be generated from an input file, but there are # multiple files, use the first file in the list for generating the output # filename. inputFile = arguments[task][inputArgument][0].split('/')[-1] inputExtension = '' for extension in argumentInformation[tool][inputArgument]['extension'].split('|'): if inputFile.endswith(extension): inputExtension = extension break # Strip off the extension. if inputExtension != '': inputFile = inputFile[0:(len(inputFile) - len(extension) - 1)] # Add the output extension unless the output is a stub. isStub = argumentInformation[tool][outputFile]['stub'] if 'stub' in argumentInformation[tool][outputFile] else False if isStub: arguments[task][outputFile].append(inputFile) else: arguments[task][outputFile].append(inputFile + '.' + argumentInformation[tool][outputFile]['extension'].split('|')[0])
def infodisc(web): print " [!] Module Selected : Information Disclosure\n\n" infodiscban() print '' time.sleep(0.3) v = raw_input('' + GR + ' [#] \033[1;4mTID\033[0m' + GR + ' :> ' + color.END) print '' if v == '1': print C + ' [!] Type Selected :' + B + ' Credit Card Enumeration' credit(web) print '\n\n' time.sleep(2) infodisc(web) elif v == '2': print C + ' [!] Type Selected :' + B + ' Extract All Emails' emailext(web) print '\n\n' time.sleep(2) infodisc(web) elif v == '3': print C + ' [!] Type Selected :' + B + ' Enumerate Errors + FPD' errors(web) print '\n\n' time.sleep(2) infodisc(web) elif v == '4': print C + ' [!] Type Selected :' + B + ' Internal IP disclosure' internalip(web) print '\n\n' time.sleep(2) infodisc(web) elif v == '5': print C + ' [!] Type Selected :' + B + ' Phone Numbers Extract' phone(web) print '\n\n' time.sleep(2) infodisc(web) elif v == '6': print C + ' [!] Type Selected :' + B + ' Social Security Numbers' ssn(web) print '\n\n' time.sleep(2) infodisc(web) elif v == 'A': print C + ' [!] Type Selected :' + B + ' All Modules' time.sleep(0.5) print C + ' [*] Firing up module -->' + B + ' Credit Cards' credit(web) print C + ' [!] Module Completed -->' + B + ' Credit Cards\n' time.sleep(1) print C + ' [*] Firing up module -->' + B + ' Email Extraction' emailext(web) print C + ' [!] Module Completed -->' + B + ' Email Hunt\n' time.sleep(1) print C + ' [*] Firing up module -->' + B + ' Errors Enumeration + FPD' errors(web) print C + ' [!] Module Completed -->' + B + ' Errors Enumeration\n' time.sleep(1) print C + ' [*] Firing up module -->' + B + ' Extract Phone Numbers' phone(web) print C + ' [!] Module Completed -->' + B + ' Extract Phone Numbers\n' time.sleep(1) print C + ' [*] Firing up module -->' + B + ' Extract Social Security Numbers' ssn(web) print C + ' [!] Module Completed -->' + B + ' Extract SSN\n' time.sleep(1) print C + ' [!] All scantypes have been tested on target...' time.sleep(1) print C + ' [*] Going back to menu...' footprintban1() footprint(web) elif v == '99': print C + ' [*] Back to the menu !' footprintban1() footprint(web) else: dope = [ 'You high dude?', 'Shit! Enter a valid option', 'Whoops! Thats not an option', 'Sorry! You just typed shit' ] print dope[randint(0, 3)] time.sleep(0.7) os.system('clear') infodisc(web)
def generateCommand(self, argumentInformation, delimiters, precommands, executables, modifiers, argumentOrder, taskToTool, linkage, taskBlock, verbose): er = errors() useStdout = False lineStart = "\t" firstCommand = True for task, path in zip(taskBlock, self.pathList): if firstCommand: if len(taskBlock) == 1: print("\t@echo -e \"Executing task: ", taskBlock[-1], '...\\c"', sep = '', file = self.makeFilehandle) else: print("\t@echo -e \"Executing piped tasks (", taskBlock[0], ' --> ', taskBlock[-1], ')...\\c"', sep = '', file = self.makeFilehandle) tool = taskToTool[task] delimiter = delimiters[tool] if tool in delimiters else ' ' # Print out the command line. print(lineStart, end = '', file = self.makeFilehandle) # If this is a single command, or the first command in a set of piped commands, # start the line with the '@' character. This will stop make 'echoing' the command # to the screen. if firstCommand: print('@', end = '', file = self.makeFilehandle) # Write the command line. if tool in precommands: print(precommands[tool], ' ', sep = '', end = '', file = self.makeFilehandle) executable = executables[tool] if path == '': print(executable, sep = '', end = '', file = self.makeFilehandle) else: print("$(", path, ")/", executable, sep = '', end = '', file = self.makeFilehandle) if tool in modifiers: print(' ', modifiers[tool], sep = '', end = '', file = self.makeFilehandle) # Check if the order in which the parameters appear matters. For certain # tools, the command line is built up of a list of arguments, but each # argument isn't given an option. For example, renaming a file has a # command line 'mv A B', where A and B are filenames that have to appear # in the correct order. toolArgumentOrder = [] if tool in argumentOrder: for argument in argumentOrder[tool]: if argument in self.arguments[task]: if self.arguments[task][argument] != '': toolArgumentOrder.append(argument) else: for argument in self.arguments[task]: toolArgumentOrder.append(argument) #FIXME CHECK ON THIS SECTION # argumentDict = {} # for argument in tl.toolInfo[tool]['arguments']: argumentDict[argument] = True # if 'argument order' in tl.toolInfo[tool]: # for argument in tl.toolInfo[tool]['argument order']: # if argument in tl.toolArguments[task]: # argumentOrder.append(argument) # del(argumentDict[argument]) # else: # self.error = False # # # If the argument is for an input or output, it is possible that these arguments # # have been removed if dealing with the stream. Check if this is the case and # # if so, do not terminate. # if 'if output to stream' in tl.toolInfo[tool]['arguments'][argument]: # if tl.toolInfo[tool]['arguments'][argument]['if output to stream'] == 'do not include': del(argumentDict[argument]) # else: self.error = True # elif 'if input is stream' in tl.toolInfo[tool]['arguments'][argument]: # if tl.toolInfo[tool]['arguments'][argument]['if input is stream'] == 'do not include': del(argumentDict[argument]) # else: self.error = True # else: self.error = True # if self.error: # er.unknownArgumentInArgumentOrder(newLine, filename, tool, argument) # er.terminate() # # # Having stepped through all of the arguments included in the ordered list, check # # if there are any arguments left in the argumentDict dictionary. This dictionary # # was built from all of the arguments for this tool and if it is not empty, then # # some of the arguments for this tool are not included in the list and thus would # # not be included in the command line. Terminate with an error if this is the case, # if len(argumentDict) != 0: # #TODO Check this error. # er.missingArgumentsInOrderList(newLine, task, tool, argumentDict) # er.terminate() # else: # for argument in tl.toolArguments[task]: argumentOrder.append(argument) #FIXME END OF SECTION TO CHECK # Include all of the set options. for argument in toolArgumentOrder: # Check if the option is a flag. isFlag = False # Check if this argument is in the toolInfo structure. Arguments dealing with # the stream may not be contained. inInfo = True if argument in argumentInformation[tool] else False isReplace = False if tool in self.addedInformation: if argument in self.addedInformation[tool]: isReplace = True #if argumentInformation[tool][argument] == 'replacement': isReplace = True if argument == 'json parameters': # If a json file is being used to generate parameters, the configuration file # needs to contain a 'json block' argument. This tells the script the name of # the block in the json file from which to extract parameters. jsonBlock = linkage[task]['json parameters']['json block'] print(" \\\n\t`python $(GKNO_PATH)/getParameters.py ", end = '', file = self.makeFilehandle) print(self.arguments[task][argument][0], ' ', jsonBlock, '`', sep = '', end = '', file = self.makeFilehandle) # If the argument is a replacement to handle a stream, do not interrogate the # tl.toolInfo structure as it doesn't contain the required values (the other # entries contain information on flags, data types etc). elif isReplace: # If the line is blank, do not print to file. if not ((argument == '') and (self.arguments[task][argument] == '')): if self.arguments[task][argument][0] == 'no value': print(" \\\n\t", argument, sep = '', end = '', file = self.makeFilehandle) else: print(" \\\n\t", argument, delimiter, self.arguments[task][argument][0], sep = '', end = '', file = self.makeFilehandle) else: # Some tools do not require a --argument or -argument in front of each value, # but just demand a particular order. The configuration file contains an # argument for these for bookkeeping purposes, but they should not be printed # to the command line. Also, commands can be included for instructing gkno to # print to stdout (e.g. use '>') or stderr ('2>'). hideArgument = False writeNothing = False useStdout = False useStderr = False if inInfo: if 'modify argument name on command line' in argumentInformation[tool][argument]: hideArgument = True if argumentInformation[tool][argument]['modify argument name on command line'] == 'hide' else False useStdout = True if argumentInformation[tool][argument]['modify argument name on command line'] == 'stdout' else False useStderr = True if argumentInformation[tool][argument]['modify argument name on command line'] == 'stderr' else False writeNothing = True if argumentInformation[tool][argument]['modify argument name on command line'] == 'omit' else False if argumentInformation[tool][argument]['type'] == 'flag': isFlag = True # If the command is a flag, check if the value is 'set' or 'unset'. If 'set', # write out the command. if isFlag: if self.arguments[task][argument][0] == 'set': print(" \\\n\t", argument, sep = '', end = '', file = self.makeFilehandle) else: # Some command lines allow multiple options to be set and the command line can # therefore be repeated multiple times. If the defined value is a list, this # is the case and the command should be written out once for each value in the # list. for value in self.arguments[task][argument]: if hideArgument: print(" \\\n\t", value, sep = '', end = '', file = self.makeFilehandle) elif writeNothing: pass elif useStdout: print(" \\\n\t> ", value, sep = '', end = '', file = self.makeFilehandle) elif useStderr: print(" \\\n\t2> ", value, sep = '', end = '', file = self.makeFilehandle) else: print(" \\\n\t", argument, delimiter, value, sep = '', end = '', file = self.makeFilehandle) if task != taskBlock[-1]: lineStart = " \\\n\t| " firstCommand = False # Write the stdout and stderr to file. print(' \\', file = self.makeFilehandle) if not useStdout: print("\t", self.redirect, ' ', self.outputID, '.stdout \\', sep = '', file = self.makeFilehandle) print("\t2", self.redirect, ' ', self.outputID, '.stderr', sep = '', file = self.makeFilehandle) print("\t@echo -e \"completed successfully.\"", sep = '', file = self.makeFilehandle)
Sest, Aest, Oest = trRGMCA(X, Aori, Afix=1) #Oracle if np.linalg.norm(Oest) == 0: Oest = X - Aest.dot(Sest) SDRMinAW[indexParam, indexIte], SDRMedAW[ indexParam, indexIte], SIRMinAW[indexParam, indexIte], SIRMedAW[ indexParam, indexIte], SNRMinAW[indexParam, indexIte], SNRMedAW[ indexParam, indexIte], SARMinAW[indexParam, indexIte], SARMedAW[ indexParam, indexIte], OmseAW[indexParam, indexIte], deltaAW[ indexParam, indexIte], angle5MaxAW[ indexParam, indexIte], angle5MedAW[indexParam, indexIte] = errors( Oest, Sest, Aest, outL, S, Aori) plt.figure(figsize=(8, 4)) plt.plot(Stransf(S)[0], 'r', linewidth=2, label=r'Initial $S_1\Phi_S^T$') plt.plot(Stransf(Sest)[0], 'g--', linewidth=2, label='Estimated Oracle') plt.legend(loc=0) plt.show() plt.figure(figsize=(8, 4)) plt.plot(outL[0], 'r', linewidth=2, label=r'Initial $O_1\Phi_O^T$') plt.plot(Oest[0], 'g--', linewidth=2, label='Estimated Oracle')
def determinePiping(arguments, argumentInformation, toolsDemandingInputStream, toolsDemandingOutputStream, workflow, taskToTool, toolsOutputtingToStream, verbose): er = errors() addArguments = {} addedInformation = {} hasPipes = False streamedOutputs = {} if len(toolsOutputtingToStream) != 0: hasPipes = True for taskCounter, task in enumerate(workflow): if task in toolsOutputtingToStream: streamedOutputs[task] = True tool = taskToTool[task] canOutputToStream = False # If the tool only deals with the stream for inputs and outputs, skip the # following search. There is no input or output commands as the tool is # expecting to be receiving and outputting to the stream. if tool in toolsDemandingOutputStream: canOutputToStream = True if not canOutputToStream: # Check if the tool allows the output to be sent to a stream. for argument in argumentInformation[tool]: isOutput = argumentInformation[tool][argument]['output'] if isOutput: if 'if output to stream' in argumentInformation[tool][argument]: if canOutputToStream: er.multipleOutputsToStream(True, "\t", task, tool) er.terminate() canOutputToStream = True # If the entry in the configuration file is 'do not include', just # remove this argument from the toolArguments structure. if argumentInformation[tool][argument]['if output to stream'] == 'do not include': del(arguments[task][argument]) # Otherwise, handle as appropriate. else: print('NOT YET HANDLED THIS STREAM OPTION', task, argument, file = sys.stdout) er.terminate() # Now check that the subsequent tool can accept the stream as an input. If # this was the last task in the pipeline, fail as the output needs to pipe # somewhere. if (taskCounter + 1) == len(workflow): er.lastTaskOutputsToPipe(True, "\t", task) er.terminate() nextTask = workflow[taskCounter + 1] nextTool = taskToTool[nextTask] canAcceptStream = False # If the current tool outputs to the stream, we need to check that the following # tool is set up to handle the stream. This information is contained in the # information for one of the input files in the next tool. If the next tool # demands the stream, then there are no input or output command line arguments as # the stream is assumed and so this check is unnecessary. if nextTool in toolsDemandingInputStream: canAcceptStream = True for argument in argumentInformation[nextTool]: isInput = argumentInformation[nextTool][argument]['input'] if isInput: if 'if input is stream' in argumentInformation[nextTool][argument]: if canAcceptStream: er.multipleInputsAcceptStream(True, "\t", task, tool) er.terminate() canAcceptStream = True # If the entry in the configuration file is 'do not include', just # remove this argument from the toolArguments structure. if argumentInformation[nextTool][argument]['if input is stream'] == 'do not include': del(arguments[nextTask][argument]) # If the entry is 'replace', then the argument needs to be removed and # replaced with that provided. When the Makefile is generated, the # tl.toolInfo structure is interogated. This replacement value should # not be present in the structure, so a value needs to be input. elif argumentInformation[nextTool][argument]['if input is stream'] == 'replace': del(arguments[nextTask][argument]) replacementArgument = argumentInformation[nextTool][argument]['replace argument with']['argument'] replacementValue = argumentInformation[nextTool][argument]['replace argument with']['value'] if replacementArgument not in arguments[nextTask]: arguments[nextTask][replacementArgument] = [] arguments[nextTask][replacementArgument].append(replacementValue) if replacementArgument in argumentInformation[nextTool]: er.replacementArgumentAlreadyPresent(True, "\t", nextTask, tool, replacementArgument) er.terminate() else: if nextTool not in addArguments: addArguments[nextTool] = {} addArguments[nextTool] = replacementArgument # If the entry is neither 'do not include' or 'replace', just use this value # as the value for the argument. else: arguments[nextTask][argument] = [] arguments[nextTask][argument].append(argumentInformation[nextTool][argument]['if input is stream']) # If no instructions are provided on how to handle a streaming input, terminate. if not canOutputToStream: er.noOutputStreamInstructions(True, "\t", task, tool) er.terminate() if not canAcceptStream: er.noInputStreamInstructions(True, "\t", nextTask, nextTool) er.terminate() # Having determined all arguments that are modified, add the necessary arguments to the # tl.toolInfo structure (these weren't added before as the dictionary cannot be modified # while it was being used). Keep track of added arguments as they need to be removed # once the makefile has been created, to reset the toolInfo structure back to its original # form before rerunning the pipeline if multiple runs are being performed. for task in addArguments: if task not in addedInformation: addedInformation[task] = [] addedInformation[task].append(addArguments[task]) return hasPipes, addedInformation
def __init__(self, tl, admin): self.arguments = {} self.argumentList = [] self.errors = errors() self.linkedArguments = {} self.uniqueArguments = {}
im_err._setError(-5) im_err.handleError() if params['fittingApertureY'] % 2 == 0: # fittingApertureY must be odd im_err._setError(-6) im_err.handleError() logging.basicConfig(format='%(levelname)s: %(message)s', level=getattr(logging, params['logLevel'].upper())) ## GET ENVIRONMENT VARS AND SET PATHS # get L2 environment variables L2_bin_dir_path = os.getenv("L2_BIN_DIR") # set L2 paths L2_bin_frpeakfinder = L2_bin_dir_path + "/frpeakfinder" im_err = errors() arcFile = FITSFile(params['pathToArcFile'], im_err) if arcFile.openFITSFile(): if not arcFile.getHeaders(0): im_err._setError(-2) im_err.handleError() if not arcFile.getData(0): im_err._setError(-3) im_err.handleError() else: im_err._setError(-1) im_err.handleError() ## L2 ## process with frpeakfinder logging.info("(__main__) executing frpeakfinder")
def determineDependencies(argumentInformation, workflow, taskToTool, toolsOutputtingToStream, arguments): er = errors() previousTask = '' dependencies = {} outputs = {} for task in workflow: tool = taskToTool[task] dependencies[task] = [] outputs[task] = [] for argument in arguments[task]: if argument == 'json parameters': value = arguments[task][argument][0] dependencies[task].append(value) else: # Check if the file is an input or an output file, or is listed as a dependent # file. If it is an output, the file should be added to the string containing # all outputs from this tool. If it is an input or dependent file, this will # be added to the string containing all files required for this tool to run. isInput = argumentInformation[tool][argument]['input'] isOutput = argumentInformation[tool][argument]['output'] isResource = argumentInformation[tool][argument]['resource'] isDependent = argumentInformation[tool][argument]['dependent'] isFlag = True if argumentInformation[tool][argument]['type'] == 'flag' else False # Determine if the input and output from this task are the stream. If so, the # dependencies and outputs structures do not need to be updated to include these # arguments outputToStream = True if task in toolsOutputtingToStream else False inputIsStream = True if (previousTask != '') and (previousTask in toolsOutputtingToStream) else False if isInput or isDependent or isOutput or isResource: # If the input/output file is defined, check that the extension is as expected. value = arguments[task][argument][0] # If this file needs to be added to one of the string, check to see if it is a stub # or not. If so, all of the files associated with the stub need to be added to the # string. isStub = argumentInformation[tool][argument]['stub'] if 'stub' in argumentInformation[tool][argument] else False # If this is a stub, create the string containing all of the files. if isStub: # Do not add the output to the self.outputs structure if the task is outputting # to the stream. Only add these values if it has been defined (i.e. value is not # and empty string). if value != '': for name in argumentInformation[tool][argument]['outputs']: if isOutput: if not outputToStream: outputs[task].append(value + name) elif isInput: if not inputIsStream: dependencies[task].append(value + name) # If the filename is not a stub, just include the value. else: if (value != '') and not isFlag: if isOutput: if not outputToStream: outputs[task].append(value) elif isInput: if not inputIsStream: dependencies[task].append(value) # Update the previous task and previous tool to be the task and tool just evaluated. previousTask = task previousTool = tool return dependencies, outputs
def checkParameters(gknoHelp, task, tool, argumentInformation, arguments, isPipeline, workflow, toolsOutputtingToStream, links, checkRequired, verbose): er = errors() for argument in arguments[task]: if argument == 'json parameters': continue # Check if the argument has been given a value. isArgumentSet = False if len(arguments[task][argument]) == 0 else True # If the argument is an input file (not from the stream) or an output file, # check that it does not contain the ':' character as this is a special # character in the Makefile. If it does, replace the ':' with a '_'. isInput = argumentInformation[tool][argument]['input'] isOutput = argumentInformation[tool][argument]['output'] if isInput or isOutput: modifiedList = [] for value in arguments[task][argument]: if ':' in value: modifiedList.append(value.replace(':', '_')) else: modifiedList.append(value) arguments[task][argument] = deepcopy(modifiedList) # If the value is required and no value has been provided, terminate. if checkRequired and not isArgumentSet: # Check if the argument is an input file and if so, if the input is coming from # the stream. If so, this does not need to be set. inputIsStream = False if isInput: previousTask = '' for currentTask in workflow: if currentTask == task: break else: previousTask = currentTask if previousTask in toolsOutputtingToStream: inputIsStream = True # Find the short form of the argument if one exists. shortForm = argumentInformation[tool][argument]['short form argument'] if 'short form argument' in argumentInformation[tool][argument] else '' # If the input to task is not a stream and the input is not set, # terminate with an error. if not inputIsStream: pipelineArgument = '' pipelineShortForm = '' if task in links: if argument in links[task]: pipelineArgument = links[task][argument][0] pipelineShortForm = links[task][argument][1] er.missingRequiredValue(verbose, task, argument, shortForm, isPipeline, pipelineArgument, pipelineShortForm) er.terminate() # If the input to this task is a stream, check if this particular argument has # instructions on how to handle the stream. If it doesn't have any, then this # input still needs to be set. else: ignoreInput = True if 'input is stream' in argumentInformation[tool][argument] else False if not ignoreInput: pipelineArgument = '' pipelineShortForm = '' if task in links: if argument in links[task]: pipelineArgument = links[task][argument][0] pipelineShortForm = links[task][argument][1] er.missingRequiredValue(verbose, task, argument, shortForm, isPipeline, pipelineArgument, pipelineShortForm) er.terminate()
def constructFilename(task, tool, argument, constructFilenames, arguments, argumentInformation, taskToTool, verbose): er = errors() basename = '' construction = '' separator = '_' filenameRoot = constructFilenames[task][argument]['filename root'] additionalText = constructFilenames[task][argument]['additional text from parameters'] if 'additional text from parameters' in \ constructFilenames[task][argument] else '' # If the basename is 'from argument', the 'tool', 'argument' and 'remove extension' # variables must be set. Take the value from the specified tool/argument, remove # the extension if requested, then add the additional text if there is any to be # added. Remove the path if one exists. if filenameRoot == 'from argument': linkedTask = constructFilenames[task][argument]['get root from task'] linkedArgument = constructFilenames[task][argument]['get root from argument'] removeExtension = constructFilenames[task][argument]['remove input extension'] # If the argument for the linked task is unset, the filename cannot be constructed, so return from # the routine without having constructed the new filename. The unset filenames will be caught later # and gkno terminated if they were required. if len(arguments[linkedTask][linkedArgument]) != 1: return # If the linked argument is set (and has only one name), construct the new filename. construction = arguments[linkedTask][linkedArgument][0].split('/')[-1] if removeExtension: linkedTool = taskToTool[linkedTask] extension = argumentInformation[linkedTool][linkedArgument]['extension'] # If there is a '|' symbol in the extension, break up all the allowable extensions # and check if the name ends with any of them and if so, remove the extension. extensions = extension.split('|') for extension in extensions: if construction.endswith(extension): construction = construction[0:(len(construction) - len(extension) - 1)] break elif filenameRoot == 'from text': construction = constructFilenames[task][argument]['filename root text'] # If there is additional text to add to the name, the additionalText variable will have # a value. If this is the case, a list of tool/argument pairs needs to be provided. These # values will be added to the name separated by the separator variable. if len(additionalText) != 0: # Determine the separator to be used when joining the values together. If no separator # is provided, use '_'. if 'separator' in additionalText: separator = additionalText['separator'] # Determine the order of the variables to add. order = additionalText['order'] for field in order: if field != 'filename root' and field != 'separator': # If the text isn't 'filename root' or 'separator', then it refers to a variable used in the pipeline. additionalTask = additionalText[field]['get parameter from task'] additionalArgument = additionalText[field]['get parameter from argument'] additionalRemoveExtension = additionalText[field]['remove extension'] additionalTool = taskToTool[additionalTask] # Now get the variable if it exists. value = '' if additionalArgument in arguments[additionalTask]: value = arguments[additionalTask][additionalArgument][0] # If the parameter being used in the output filename is itself a filename, # it should contain a path. If this is the case, remove the path from the # name before using in the construction. value = value.split('/')[-1] # If the extension is to be removed, check that the input argument defines a file # with an extension and if so, remove the extension if requested. if additionalRemoveExtension: extension = argumentInformation[additionalTool][additionalArgument]['extension'] # If there is a '|' symbol in the extension, break up all the allowable extensions # and check if the name ends with any of them and if so, remove the extension. if extension != '' and extension != 'no extension': extensions = extension.split('|') for extension in extensions: if value.endswith(extension): value = value[0:(len(value) - len(extension) - 1)] break if value != '': construction += separator + str(value) # Determine if the output file is a stub. If not, add the extension to the output file. isStub = argumentInformation[tool][argument]['stub'] if 'stub' in argumentInformation[tool][argument] else False if not isStub: extension = argumentInformation[tool][argument]['extension'] # Some tools can operate on any file (e.g. gzip) and so have the extension listed as # 'no extension'. If this is the case, check if the output extension is given in the # pipeline configuration file. If so, use this extension, if not, do not add an # extension. if extension == 'no extension': outputExtension = constructFilenames[task][argument]['output extension'] if 'output extension' in constructFilenames[task][argument] else '' if outputExtension != '': construction += '.' + str(outputExtension) else: construction += '.' + str(extension) # Having built the filename, set the value in the tl.toolArguments data structure. arguments[task][argument].append(construction)
def constructFilenames(task, tool, arguments, argumentInformation, constructFilenames, links, taskToTool, verbose): er = errors() # First identify the input and output files and how their names should be constructed. filenameConstructor = '' inputFiles = [] outputFiles = {} for argument in arguments[task]: if argument != 'json parameters': isInput = argumentInformation[tool][argument]['input'] isOutput = argumentInformation[tool][argument]['output'] # If this is an input file, check to see if it is to be used for building filenames. if isInput: inputFiles.append(argument) if 'use for filenames' in argumentInformation[tool][argument]: if argumentInformation[tool][argument]['use for filenames']: # If there was an input file previous designated as the file to use for # constructing output filenames for this tool, gkno cannot determine which # file to use and so terminates. if filenameConstructor != '': er.multipleFilenameConstructors(verbose, task, tool, argument, filenameConstructor) er.terminate() else: filenameConstructor = argument # If the argument is an output, check to see if it has already been defined. If # not, store this as a filename to be constructed. if isOutput: if len(arguments[task][argument]) == 0: # For pipelines, check if the 'construct filenames' block exists and describes # how to build this filename. if task in constructFilenames: for constructArgument in constructFilenames[task]: if constructArgument == argument: outputFiles[argument] = 'construct' break # If the argument to be built has no instructions on how to construct the filename. if argument not in outputFiles: # If the configuration file specifically states not to construct the filename from # the input file, terminate as the user needs to input the output filename. Otherwise # use the input file. if 'do not construct filename from input' in argumentInformation[tool][argument]: shortForm = argumentInformation[tool][argument]['short form argument'] if 'short form argument' in \ argumentInformation[tool][argument] else '' if task in links: pipelineArgument = '' if argument in links[task]: pipelineArgument = links[task][argument][0] pipelineShortForm = links[task][argument][1] er.missingFilenameNotToBeConstructed(verbose, task, tool, argument, shortForm, pipelineArgument, pipelineShortForm) er.terminate() else: outputFiles[argument] = 'from input' # Now all of the output files have been identified and the method of filename generation # determined, build the filenames. for outputFile in outputFiles: # If to be constructed using information from the configuration file, construct. if outputFiles[outputFile] == 'construct': constructFilename(task, tool, outputFile, constructFilenames, arguments, argumentInformation, taskToTool, verbose) elif outputFiles[outputFile] == 'from input': # If the output filename is to be generated using an input file from this task and there # are no input files designated as to be used for generating the filename, check what # input files there are. If there is only one input for this task, use this file to # generate the filenames. If there are multiple or no input file arguments, terminate. if filenameConstructor == '': if len(inputFiles) == 1: constructFilenameFromInput(task, tool, argumentInformation, arguments, outputFile, inputFiles[0], verbose) #else: # er.unknownFilenameConstructor(verbose, task, tool, outputFile) # er.terminate() else: constructFilenameFromInput(task, tool, argumentInformation, arguments, outputFile, filenameConstructor, verbose)