class PegasusWorkflowModel (WorkflowModel): def __init__(self, namespace, wms): logger.debug ("wms:pegasus:create-workflowmodel: %s" % namespace) self.wms = wms logger.debug ("outputdir: %s", wms.getOutputDir ()) self.adag = ADAG (namespace) self.namespace = namespace self.files = {} self.exes = {} self.propertyMap = {} self.nodes = {} self.jobTransformations = {} self.variableMap = { 'literalToVariable' : {}, 'literalToNodeId' : {} } def addToVariableMap (self, literal, variable, id): self.variableMap ['literalToVariable'][literal] = variable self.variableMap ['literalToNodeId'][literal] = id logger.debug ("variablematch: recorded lit=%s var=%s id=%s", literal, variable, id) def setProperties (self, nodeId, properties): logger.debug ("wms:pegasus:dax:setprops: (%s)->(%s)" % (nodeId, properties)) self.propertyMap [nodeId] = properties def getProperties (self, nodeId): return self.propertyMap [nodeId] def createFile (self, fileName): return File (fileName) def addNode (self, id, node): logger.debug ("wms:pegasus:dax:add-node: (%s)->(%s)" % (node.getId(), properties)) self.nodes [id] = node def getNode (self, id): node = None if id in self.nodes: node = self.nodes [id] return node def createFile (self, fileName, fileURL=None, site=None): #traceback.print_stack () file = self.getFile (fileName) if not file: file = File (fileName) if not fileURL: fileURL = "file://%s/%s" % (self.wms.getOutputDir (), fileName) logger.debug ("fileurl: %s", fileURL) if not site: site = "local" if not isinstance(fileURL, basestring) and len (fileURL) > 0: fileURL = fileURL [0] logger.debug ("--add-pfn: (%s)(%s)(%s)", fileName, fileURL, site) pfn = PFN (fileURL, site) file.addPFN (pfn) self.files [fileName] = file return file def addFileToDAG (self, file): if not self.adag.hasFile (file): self.adag.addFile (file) def removeFileFromDAG (self, file): if file in self.adag.files: self.adag.files.remove (file) def addFile (self, fileName, fileURL=None, site=None): #traceback.print_stack () file = self.getFile (fileName) if not file: file = File (fileName) if not fileURL: fileURL = "file://%s/%s" % (self.wms.getOutputDir (), fileName) logger.debug ("fileurl: %s", fileURL) if not site: site = "local" if not isinstance(fileURL, basestring) and len (fileURL) > 0: fileURL = fileURL [0] logger.debug ("--add-pfn: (%s)(%s)(%s)", fileName, fileURL, site) pfn = PFN (fileURL, site) file.addPFN (pfn) self.adag.addFile (file) self.files [fileName] = file return file def getFile (self, fileName, prefix=""): value = None key = "%s%s" % (prefix, fileName) if key in self.files: logger.debug ("wms:pegasus:dax:get-file: (%s)" % key) value = self.files [key] else: value = None return value def addExecutable (self, jobId, name, path, version="1.0", exe_os="linux", exe_arch="x86_64", site="local", installed="true"): e_exe = self.getExecutable (name) if not version: version = "1.0" if not exe_arch: exe_arch="x86_64" if not e_exe: e_exe = Executable ( namespace=self.namespace, name=name, version=version, os=exe_os, arch=exe_arch, installed=installed) if not site: site = "local" if not installed: installed = False if logging.getLogger().isEnabledFor (logging.DEBUG): logger.debug ("wms:pegasus:dax:add-exe: (name=[%s], path=[%s], version=[%s], os=[%s], arch=[%s], site=[%s], installed=[%s])" % (name, path, version, exe_os, exe_arch, site, installed)) if not "://" in path: path = "file://%s" % path if not path: raise ValueError ("empty path for executable: %s at site %s" % (name, site)) e_exe.addPFN (PFN (path, site)) if not installed: e_exe.installed = installed self.adag.addExecutable (e_exe) self.exes [name] = e_exe transformation = Transformation (name, self.namespace, version) self.jobTransformations [jobId] = transformation return e_exe def getExecutable (self, name): key = name if key in self.exes: return self.exes [key] else: return None def addSubWorkflow (self, name, transformation=None): #self.adag.addTransformation (transformation) abstractJob = DAX (name) self.adag.addDAX (abstractJob) return abstractJob def addJob (self, id): #self.adag.addTransformation (transformation) transformation = self.jobTransformations [id] logger.debug ("wms:pegasus:add-job: transformation(%s) jobid(%s)", transformation.name, id) abstractJob = Job (name=transformation, id=id) self.adag.addJob (abstractJob) return abstractJob def addProfiles (self, abstractJob, profiles): if profiles: for astProfile in profiles: logger.debug ("wms:pegasus:add-profile: (namespace=%s,key=%s,value=%s) to job (%s)", astProfile.namespace, astProfile.key, astProfile.value, abstractJob.name) profile = Profile (astProfile.namespace, astProfile.key, astProfile.value) abstractJob.addProfile (profile) def addFiles (self, abstractJob, files, link): if files: for fileKey in files: tuple = files [fileKey] fileElement = tuple [0] file = fileElement.getDaxNode () try: isLeaf = tuple [2] if isLeaf: abstractJob.uses (file, link=link, transfer=True) else: abstractJob.uses (file, link=link) ''' abstractJob.uses (file, link=link) ''' arg = tuple [1] if arg: abstractJob.addArguments (arg, file) except DuplicateError: pass def addInputFiles (self, abstractJob, files): self.addFiles (abstractJob, files, Link.INPUT) def addOutputFiles (self, abstractJob, files): self.addFiles (abstractJob, files, Link.OUTPUT) def addArguments (self, abstractJob, arguments): if arguments: abstractJob.addArguments (arguments) def addDependency (self, parent, child): self.adag.addDependency (Dependency (parent, child)) def writeExecutable (self, stream): self.adag.writeXML (stream) filename = "%s.%s" % (self.namespace, 'obj') filepath = os.path.join (self.wms.getOutputDir (), filename) try: output = None try: output = open (filepath, 'w') output.write (json.dumps (self.variableMap, indent=3, sort_keys=True)) output.flush () finally: if output: output.close () except IOError: traceback.print_stack () def getADAG (self): return self.adag