class PegasusWorkflowModel (WorkflowModel):
def __init__(self, namespace, wms):
logger.debug ("wms:pegasus:create-workflowmodel: %s" % namespace)
self.wms = wms
logger.debug ("outputdir: %s", wms.getOutputDir ())
self.adag = ADAG (namespace)
self.namespace = namespace
self.files = {}
self.exes = {}
self.propertyMap = {}
self.nodes = {}
self.jobTransformations = {}
self.variableMap = {
'literalToVariable' : {},
'literalToNodeId' : {}
}
def addToVariableMap (self, literal, variable, id):
self.variableMap ['literalToVariable'][literal] = variable
self.variableMap ['literalToNodeId'][literal] = id
logger.debug ("variablematch: recorded lit=%s var=%s id=%s", literal, variable, id)
def setProperties (self, nodeId, properties):
logger.debug ("wms:pegasus:dax:setprops: (%s)->(%s)" % (nodeId, properties))
self.propertyMap [nodeId] = properties
def getProperties (self, nodeId):
return self.propertyMap [nodeId]
def createFile (self, fileName):
return File (fileName)
def addNode (self, id, node):
logger.debug ("wms:pegasus:dax:add-node: (%s)->(%s)" % (node.getId(), properties))
self.nodes [id] = node
def getNode (self, id):
node = None
if id in self.nodes:
node = self.nodes [id]
return node
def createFile (self, fileName, fileURL=None, site=None):
#traceback.print_stack ()
file = self.getFile (fileName)
if not file:
file = File (fileName)
if not fileURL:
fileURL = "file://%s/%s" % (self.wms.getOutputDir (), fileName)
logger.debug ("fileurl: %s", fileURL)
if not site:
site = "local"
if not isinstance(fileURL, basestring) and len (fileURL) > 0:
fileURL = fileURL [0]
logger.debug ("--add-pfn: (%s)(%s)(%s)", fileName, fileURL, site)
pfn = PFN (fileURL, site)
file.addPFN (pfn)
self.files [fileName] = file
return file
def addFileToDAG (self, file):
if not self.adag.hasFile (file):
self.adag.addFile (file)
def removeFileFromDAG (self, file):
if file in self.adag.files:
self.adag.files.remove (file)
def addFile (self, fileName, fileURL=None, site=None):
#traceback.print_stack ()
file = self.getFile (fileName)
if not file:
file = File (fileName)
if not fileURL:
fileURL = "file://%s/%s" % (self.wms.getOutputDir (), fileName)
logger.debug ("fileurl: %s", fileURL)
if not site:
site = "local"
if not isinstance(fileURL, basestring) and len (fileURL) > 0:
fileURL = fileURL [0]
logger.debug ("--add-pfn: (%s)(%s)(%s)", fileName, fileURL, site)
pfn = PFN (fileURL, site)
file.addPFN (pfn)
self.adag.addFile (file)
self.files [fileName] = file
return file
def getFile (self, fileName, prefix=""):
value = None
key = "%s%s" % (prefix, fileName)
if key in self.files:
logger.debug ("wms:pegasus:dax:get-file: (%s)" % key)
value = self.files [key]
else:
value = None
return value
def addExecutable (self, jobId, name, path, version="1.0", exe_os="linux", exe_arch="x86_64", site="local", installed="true"):
e_exe = self.getExecutable (name)
if not version:
version = "1.0"
if not exe_arch:
exe_arch="x86_64"
if not e_exe:
e_exe = Executable (
namespace=self.namespace,
name=name,
version=version,
os=exe_os,
arch=exe_arch,
installed=installed)
if not site:
site = "local"
if not installed:
installed = False
if logging.getLogger().isEnabledFor (logging.DEBUG):
logger.debug ("wms:pegasus:dax:add-exe: (name=[%s], path=[%s], version=[%s], os=[%s], arch=[%s], site=[%s], installed=[%s])" %
(name,
path,
version,
exe_os,
exe_arch,
site,
installed))
if not "://" in path:
path = "file://%s" % path
if not path:
raise ValueError ("empty path for executable: %s at site %s" % (name, site))
e_exe.addPFN (PFN (path, site))
if not installed:
e_exe.installed = installed
self.adag.addExecutable (e_exe)
self.exes [name] = e_exe
transformation = Transformation (name, self.namespace, version)
self.jobTransformations [jobId] = transformation
return e_exe
def getExecutable (self, name):
key = name
if key in self.exes:
return self.exes [key]
else:
return None
def addSubWorkflow (self, name, transformation=None):
#self.adag.addTransformation (transformation)
abstractJob = DAX (name)
self.adag.addDAX (abstractJob)
return abstractJob
def addJob (self, id):
#self.adag.addTransformation (transformation)
transformation = self.jobTransformations [id]
logger.debug ("wms:pegasus:add-job: transformation(%s) jobid(%s)", transformation.name, id)
abstractJob = Job (name=transformation, id=id)
self.adag.addJob (abstractJob)
return abstractJob
def addProfiles (self, abstractJob, profiles):
if profiles:
for astProfile in profiles:
logger.debug ("wms:pegasus:add-profile: (namespace=%s,key=%s,value=%s) to job (%s)",
astProfile.namespace,
astProfile.key,
astProfile.value,
abstractJob.name)
profile = Profile (astProfile.namespace,
astProfile.key,
astProfile.value)
abstractJob.addProfile (profile)
def addFiles (self, abstractJob, files, link):
if files:
for fileKey in files:
tuple = files [fileKey]
fileElement = tuple [0]
file = fileElement.getDaxNode ()
try:
isLeaf = tuple [2]
if isLeaf:
abstractJob.uses (file, link=link, transfer=True)
else:
abstractJob.uses (file, link=link)
'''
abstractJob.uses (file, link=link)
'''
arg = tuple [1]
if arg:
abstractJob.addArguments (arg, file)
except DuplicateError:
pass
def addInputFiles (self, abstractJob, files):
self.addFiles (abstractJob, files, Link.INPUT)
def addOutputFiles (self, abstractJob, files):
self.addFiles (abstractJob, files, Link.OUTPUT)
def addArguments (self, abstractJob, arguments):
if arguments:
abstractJob.addArguments (arguments)
def addDependency (self, parent, child):
self.adag.addDependency (Dependency (parent, child))
def writeExecutable (self, stream):
self.adag.writeXML (stream)
filename = "%s.%s" % (self.namespace, 'obj')
filepath = os.path.join (self.wms.getOutputDir (), filename)
try:
output = None
try:
output = open (filepath, 'w')
output.write (json.dumps (self.variableMap, indent=3, sort_keys=True))
output.flush ()
finally:
if output:
output.close ()
except IOError:
traceback.print_stack ()
def getADAG (self):
return self.adag
