def ExecuteWorkflow (user, archive, archivePath, workflowRoot=".", amqpSettings=None):
logger.info ("executeworkflow:amqpsettings: %s", amqpSettings)
try:
basename = os.path.basename (archive)
unpack_dir = GraysonWebUtil.form_workflow_path (user, GraysonWebConst.UNPACK_EXT % basename)
# If we don't open up permissions, using x509userproxy doesn't work. i.e. users can't run as themselves and write files here, for example.
chmodCommand = "chmod -R 777 %s" % unpack_dir
logger.debug ("setting output directory permissions: %s" % chmodCommand)
os.system (chmodCommand)
log_file = os.path.join (unpack_dir, "log.txt")
executionMonitor = WorkflowMonitorCompilerPlugin (user.username,
unpack_dir,
workflowRoot,
amqpSettings)
logger.debug ("""
=======================================================================================
== C O M P I L E (%s)
=======================================================================================
== user.username: (%s)
== outputDir : (%s)
== logFile : (%s)
== amqp : (%s)
=======================================================================================""",
archivePath,
user.username,
unpack_dir,
log_file,
amqpSettings)
GraysonCompiler.compile (models = [ archivePath ],
outputdir = unpack_dir,
logDir = unpack_dir,
appHome = unpack_dir,
toLogFile = "log.txt",
execute = True,
logLevel = "debug",
plugin = executionMonitor)
except ValueError, e:
traceback.print_exc ()
eventStream = EventStream (amqpSettings)
eventStream.sendCompilationMessagesEvent (username = user.username,
flowId = archive,
log = log_file)
def test_compile (self):
self.banner ()
modelPattern = self.getUnpack ("model", "*.graphml")
names = glob.glob (modelPattern)
output_workflow = self.getUnpack ("workflow.dax")
GraysonCompiler.compile (models = names,
output = output_workflow,
modelPath = [ self.getUnpack ("model") ],
namespace="grayson-test",
version="2.0",
logLevel = self.logLevel)
self.logger.info ("========================================")
self.logger.info ("Analyzing generated DAX with Pegasus API")
self.logger.info ("========================================")
dag = Pegasus.DAX3.parse (output_workflow)
self.logger.info ("dag-name: %s", dag.name)
self.logger.info ("dag-files:")
for file in dag.files:
self.logger.info (" %s", file.name)
self.logger.info ("dag-jobs:")
for job in dag.jobs:
if isinstance (job, DAX):
self.logger.info (" %s", job.file)
else:
self.logger.info (" %s namespace: %s version: %s", job.name, job.namespace, job.version)
self.logger.info ("dag-transformations:")
for trans in dag.transformations:
self.logger.info (" %s (namespace:%s, version:%s", trans.name, trans.namespace, trans.version)
for use in trans.used_files:
self.logger.info (" %s", use.name)
self.logger.info ("dag-executables:")
for exe in dag.executables:
self.logger.info (" %s (namespace:%s, version:%s, arch:%s, os:%s)", exe.name, exe.namespace, exe.version, exe.arch, exe.os)
self.logger.info ("dag-dependencies:")
for dep in dag.dependencies:
self.logger.info (" %s", dep.child.id)
self.assertEqual('a', 'a')
def execute (self, context={}):
operatorContext = context ['operator']
method = operatorContext ["method"]
inputFile = operatorContext ["input"]
variable = operatorContext ["variable"]
index = operatorContext ["index"]
flow = operatorContext ["flow"]
version = operatorContext ["version"]
instanceArgs = operatorContext ["instanceArgs"]
mapType = context ["mapType"]
outputBasename = context ["outputName"]
modelPath = context ["modelPath"]
outputDir = context ["outputDir"]
contextModels = context ["contextModels"]
sites = context ["sites"]
appHome = context ["appHome"]
graysonHome = context ["graysonHome"]
#print "%s" % json.dumps (context, indent=3, sort_keys=True)
tmpOutputDir = os.path.join (outputDir, "tmp") # avoid overwriting replica catalog.
contextModels = contextModels.split (os.pathsep)
models = [ flow ]
for model in contextModels:
models.append (model)
main_flow_name = os.path.join (outputDir, "%s.dax" % flow.replace (".graphml", ""))
namespace = flow.replace (".graphml", "")
flowContext = { "namespace" : outputBasename }
template = Template (self.header)
text = [ template.substitute (flowContext) ]
replicaText = []
if mapType == 'tar':
tar = tarfile.open (inputFile, "r:gz")
members = tar.getmembers ()
c = 0
for archiveMember in members:
outputname = "%s.%s.dax" % ( outputBasename, c )
definitions = {
variable : archiveMember.name,
index : "%s" % c,
Operator.DYNAMIC_INDEX : "%s" % c,
"appHome" : appHome
}
logger.debug ("dynamic-map: invoking compiler")
try:
output = open (os.path.join (outputDir, outputname), 'w')
try:
GraysonCompiler.compile (models = models,
output = output,
modelPath = modelPath.split (os.pathsep),
namespace = namespace,
version = None,
logLevel = "debug",
modelProperties = definitions,
outputdir = tmpOutputDir,
sites = sites,
toLogFile = os.path.join (outputDir, "log.txt"))
finally:
if output:
output.close ()
except IOError as e:
logger.error ("Encountered IOError %s compiling subdax %s", e.__str__ (), output)
raise e
replicaText.append ('%s file://%s/%s pool="local"' % (outputname, outputDir, outputname))
template = Template (self.subdax)
flowContext ['c'] = c
flowContext ['outputname'] = outputname
flowContext ['instanceArgs'] = instanceArgs
flowContext ['sites'] = "--sites %s" % sites if instanceArgs == "" else ""
text.append (template.substitute (flowContext))
replicaCatalogName = "replica-catalog.rc"
masterRC = os.path.join (outputDir, replicaCatalogName)
self.updateCatalog (master = masterRC,
other = os.path.join (tmpOutputDir, replicaCatalogName))
c += 1
elif mapType == 'list':
stream = open (inputFile, "r")
c = 0
for line in stream:
outputname = "%s.%s.dax" % ( outputBasename, c )
definitions = {
variable : line,
index : "%s" % c,
Operator.DYNAMIC_INDEX : "%s" % c,
"appHome" : appHome
}
logger.debug ("dynamic-map: invoking compiler")
try:
output = open (os.path.join (outputDir, outputname), 'w')
try:
GraysonCompiler.compile (models = models,
output = output,
modelPath = modelPath.split (os.pathsep),
namespace = namespace,
version = None,
logLevel = "debug",
modelProperties = definitions,
outputdir = tmpOutputDir,
sites = sites,
toLogFile = os.path.join (outputDir, "log.txt"))
finally:
if output:
output.close ()
except IOError as e:
logger.error ("Encountered IOError %s compiling subdax %s", e.__str__ (), output)
raise e
replicaText.append ('%s file://%s/%s pool="local"' % (outputname, outputDir, outputname))
template = Template (self.subdax)
flowContext ['c'] = c
flowContext ['outputname'] = outputname
flowContext ['instanceArgs'] = instanceArgs
flowContext ['sites'] = "--sites %s" % sites if instanceArgs == "" else ""
text.append (template.substitute (flowContext))
replicaCatalogName = "replica-catalog.rc"
masterRC = os.path.join (outputDir, replicaCatalogName)
self.updateCatalog (master = masterRC,
other = os.path.join (tmpOutputDir, replicaCatalogName))
c += 1
text.append (self.footer)
mainFlowContent = ''.join (text)
GraysonUtil.writeFile (outputPath = os.path.join (outputDir, main_flow_name),
data = mainFlowContent)
logger.debug ("dynamic-map: writing output dax: %s" % mainFlowContent)
replicaText.append ('%s file://%s pool="local"' % (os.path.basename (main_flow_name), main_flow_name))
GraysonUtil.writeFile (os.path.join (outputDir, "tmp", replicaCatalogName),
'\n'.join (replicaText))
self.updateCatalog (master = masterRC,
other = os.path.join (tmpOutputDir, replicaCatalogName))
transformationCatalogName = "transformation-catalog.tc"
masterTC = os.path.join (outputDir, transformationCatalogName)
self.updateCatalog (master = masterTC,
other = os.path.join (tmpOutputDir, transformationCatalogName))
