def updateCatalog (self, masterCat, other):
newResources = []
master = GraysonUtil.readFile (masterCat)
masterLines = master.split ('\n')
for line in masterLines:
logger.debug (" __ _ _ _ _ _ _____: %s", line)
sub = GraysonUtil.readFile (other)
subLines = sub.split ('\n')
masterMap = {}
for line in masterLines:
parts = line.split (' ')
if len (parts) > 0:
resource = parts [0]
if resource:
masterMap [resource] = resource
logger.debug (" resource : %s", resource)
for line in subLines:
parts = line.split (' ')
if len (parts) > 0:
resource = parts [0]
if resource and not resource in masterMap:
newResources.append (line)
logger.debug (" new resource: %s", resource)
GraysonUtil.writeFile (masterCat, "%s\n%s" % (master, '\n'.join (newResources)))
def writeMetaDataCatalogs (self):
GraysonUtil.writeFile (
outputPath=os.path.join (self.outputDir, PegasusProperties.SITE_CATALOG),
data=self.getSiteCatalog().generateXML ())
''' replica catalog '''
GraysonUtil.writeFile (
outputPath=os.path.join (self.outputDir, PegasusProperties.REPLICA_CATALOG),
data=self.getReplicaCatalog().generateRC ())
''' transformation catalog '''
GraysonUtil.writeFile (
outputPath=os.path.join (self.outputDir, PegasusProperties.TRANSFORMATION_CATALOG),
data=self.getTransformationCatalog().generateTC ())
''' properties '''
GraysonUtil.writeFile (
outputPath=os.path.join (os.path.join (self.outputDir, PegasusProperties.PEGASUS_PROPERTIES)),
data=self.pegasusProperties.generateProperties (configDir=self.outputDir))
def put_file (request):
path = request.REQUEST ["path"]
content = request.REQUEST ["content"]
logger.debug ("writing file: %s", path)
GraysonUtil.writeFile (path, content)
return ViewUtil.get_json_response ({ "status" : "ok" })
def writeFlows (self, root, flows):
fileName = self.flowPath (root)
workflowText = json.dumps (flows, indent=3, sort_keys=True)
logger.debug (workflowText)
GraysonUtil.writeFile (fileName, workflowText)
def execute (self, context={}):
operatorContext = context ['operator']
method = operatorContext ["method"]
inputFile = operatorContext ["input"]
variable = operatorContext ["variable"]
index = operatorContext ["index"]
flow = operatorContext ["flow"]
version = operatorContext ["version"]
instanceArgs = operatorContext ["instanceArgs"]
mapType = context ["mapType"]
outputBasename = context ["outputName"]
modelPath = context ["modelPath"]
outputDir = context ["outputDir"]
contextModels = context ["contextModels"]
sites = context ["sites"]
appHome = context ["appHome"]
graysonHome = context ["graysonHome"]
#print "%s" % json.dumps (context, indent=3, sort_keys=True)
tmpOutputDir = os.path.join (outputDir, "tmp") # avoid overwriting replica catalog.
contextModels = contextModels.split (os.pathsep)
models = [ flow ]
for model in contextModels:
models.append (model)
main_flow_name = os.path.join (outputDir, "%s.dax" % flow.replace (".graphml", ""))
namespace = flow.replace (".graphml", "")
flowContext = { "namespace" : outputBasename }
template = Template (self.header)
text = [ template.substitute (flowContext) ]
replicaText = []
if mapType == 'tar':
tar = tarfile.open (inputFile, "r:gz")
members = tar.getmembers ()
c = 0
for archiveMember in members:
outputname = "%s.%s.dax" % ( outputBasename, c )
definitions = {
variable : archiveMember.name,
index : "%s" % c,
Operator.DYNAMIC_INDEX : "%s" % c,
"appHome" : appHome
}
logger.debug ("dynamic-map: invoking compiler")
try:
output = open (os.path.join (outputDir, outputname), 'w')
try:
GraysonCompiler.compile (models = models,
output = output,
modelPath = modelPath.split (os.pathsep),
namespace = namespace,
version = None,
logLevel = "debug",
modelProperties = definitions,
outputdir = tmpOutputDir,
sites = sites,
toLogFile = os.path.join (outputDir, "log.txt"))
finally:
if output:
output.close ()
except IOError as e:
logger.error ("Encountered IOError %s compiling subdax %s", e.__str__ (), output)
raise e
replicaText.append ('%s file://%s/%s pool="local"' % (outputname, outputDir, outputname))
template = Template (self.subdax)
flowContext ['c'] = c
flowContext ['outputname'] = outputname
flowContext ['instanceArgs'] = instanceArgs
flowContext ['sites'] = "--sites %s" % sites if instanceArgs == "" else ""
text.append (template.substitute (flowContext))
replicaCatalogName = "replica-catalog.rc"
masterRC = os.path.join (outputDir, replicaCatalogName)
self.updateCatalog (master = masterRC,
other = os.path.join (tmpOutputDir, replicaCatalogName))
c += 1
elif mapType == 'list':
stream = open (inputFile, "r")
c = 0
for line in stream:
outputname = "%s.%s.dax" % ( outputBasename, c )
definitions = {
variable : line,
index : "%s" % c,
Operator.DYNAMIC_INDEX : "%s" % c,
"appHome" : appHome
}
logger.debug ("dynamic-map: invoking compiler")
try:
output = open (os.path.join (outputDir, outputname), 'w')
try:
GraysonCompiler.compile (models = models,
output = output,
modelPath = modelPath.split (os.pathsep),
namespace = namespace,
version = None,
logLevel = "debug",
modelProperties = definitions,
outputdir = tmpOutputDir,
sites = sites,
toLogFile = os.path.join (outputDir, "log.txt"))
finally:
if output:
output.close ()
except IOError as e:
logger.error ("Encountered IOError %s compiling subdax %s", e.__str__ (), output)
raise e
replicaText.append ('%s file://%s/%s pool="local"' % (outputname, outputDir, outputname))
template = Template (self.subdax)
flowContext ['c'] = c
flowContext ['outputname'] = outputname
flowContext ['instanceArgs'] = instanceArgs
flowContext ['sites'] = "--sites %s" % sites if instanceArgs == "" else ""
text.append (template.substitute (flowContext))
replicaCatalogName = "replica-catalog.rc"
masterRC = os.path.join (outputDir, replicaCatalogName)
self.updateCatalog (master = masterRC,
other = os.path.join (tmpOutputDir, replicaCatalogName))
c += 1
text.append (self.footer)
mainFlowContent = ''.join (text)
GraysonUtil.writeFile (outputPath = os.path.join (outputDir, main_flow_name),
data = mainFlowContent)
logger.debug ("dynamic-map: writing output dax: %s" % mainFlowContent)
replicaText.append ('%s file://%s pool="local"' % (os.path.basename (main_flow_name), main_flow_name))
GraysonUtil.writeFile (os.path.join (outputDir, "tmp", replicaCatalogName),
'\n'.join (replicaText))
self.updateCatalog (master = masterRC,
other = os.path.join (tmpOutputDir, replicaCatalogName))
transformationCatalogName = "transformation-catalog.tc"
masterTC = os.path.join (outputDir, transformationCatalogName)
self.updateCatalog (master = masterTC,
other = os.path.join (tmpOutputDir, transformationCatalogName))
def updateCatalog (self, master, other):
masterText = GraysonUtil.readFile (master)
text = GraysonUtil.readFile (other)
GraysonUtil.writeFile (master, "%s\n%s" % (masterText, text))
