def ExecuteWorkflow (user, archive, archivePath, workflowRoot=".", amqpSettings=None): logger.info ("executeworkflow:amqpsettings: %s", amqpSettings) try: basename = os.path.basename (archive) unpack_dir = GraysonWebUtil.form_workflow_path (user, GraysonWebConst.UNPACK_EXT % basename) # If we don't open up permissions, using x509userproxy doesn't work. i.e. users can't run as themselves and write files here, for example. chmodCommand = "chmod -R 777 %s" % unpack_dir logger.debug ("setting output directory permissions: %s" % chmodCommand) os.system (chmodCommand) log_file = os.path.join (unpack_dir, "log.txt") executionMonitor = WorkflowMonitorCompilerPlugin (user.username, unpack_dir, workflowRoot, amqpSettings) logger.debug (""" ======================================================================================= == C O M P I L E (%s) ======================================================================================= == user.username: (%s) == outputDir : (%s) == logFile : (%s) == amqp : (%s) =======================================================================================""", archivePath, user.username, unpack_dir, log_file, amqpSettings) GraysonCompiler.compile (models = [ archivePath ], outputdir = unpack_dir, logDir = unpack_dir, appHome = unpack_dir, toLogFile = "log.txt", execute = True, logLevel = "debug", plugin = executionMonitor) except ValueError, e: traceback.print_exc () eventStream = EventStream (amqpSettings) eventStream.sendCompilationMessagesEvent (username = user.username, flowId = archive, log = log_file)
def test_compile (self): self.banner () modelPattern = self.getUnpack ("model", "*.graphml") names = glob.glob (modelPattern) output_workflow = self.getUnpack ("workflow.dax") GraysonCompiler.compile (models = names, output = output_workflow, modelPath = [ self.getUnpack ("model") ], namespace="grayson-test", version="2.0", logLevel = self.logLevel) self.logger.info ("========================================") self.logger.info ("Analyzing generated DAX with Pegasus API") self.logger.info ("========================================") dag = Pegasus.DAX3.parse (output_workflow) self.logger.info ("dag-name: %s", dag.name) self.logger.info ("dag-files:") for file in dag.files: self.logger.info (" %s", file.name) self.logger.info ("dag-jobs:") for job in dag.jobs: if isinstance (job, DAX): self.logger.info (" %s", job.file) else: self.logger.info (" %s namespace: %s version: %s", job.name, job.namespace, job.version) self.logger.info ("dag-transformations:") for trans in dag.transformations: self.logger.info (" %s (namespace:%s, version:%s", trans.name, trans.namespace, trans.version) for use in trans.used_files: self.logger.info (" %s", use.name) self.logger.info ("dag-executables:") for exe in dag.executables: self.logger.info (" %s (namespace:%s, version:%s, arch:%s, os:%s)", exe.name, exe.namespace, exe.version, exe.arch, exe.os) self.logger.info ("dag-dependencies:") for dep in dag.dependencies: self.logger.info (" %s", dep.child.id) self.assertEqual('a', 'a')
def execute (self, context={}): operatorContext = context ['operator'] method = operatorContext ["method"] inputFile = operatorContext ["input"] variable = operatorContext ["variable"] index = operatorContext ["index"] flow = operatorContext ["flow"] version = operatorContext ["version"] instanceArgs = operatorContext ["instanceArgs"] mapType = context ["mapType"] outputBasename = context ["outputName"] modelPath = context ["modelPath"] outputDir = context ["outputDir"] contextModels = context ["contextModels"] sites = context ["sites"] appHome = context ["appHome"] graysonHome = context ["graysonHome"] #print "%s" % json.dumps (context, indent=3, sort_keys=True) tmpOutputDir = os.path.join (outputDir, "tmp") # avoid overwriting replica catalog. contextModels = contextModels.split (os.pathsep) models = [ flow ] for model in contextModels: models.append (model) main_flow_name = os.path.join (outputDir, "%s.dax" % flow.replace (".graphml", "")) namespace = flow.replace (".graphml", "") flowContext = { "namespace" : outputBasename } template = Template (self.header) text = [ template.substitute (flowContext) ] replicaText = [] if mapType == 'tar': tar = tarfile.open (inputFile, "r:gz") members = tar.getmembers () c = 0 for archiveMember in members: outputname = "%s.%s.dax" % ( outputBasename, c ) definitions = { variable : archiveMember.name, index : "%s" % c, Operator.DYNAMIC_INDEX : "%s" % c, "appHome" : appHome } logger.debug ("dynamic-map: invoking compiler") try: output = open (os.path.join (outputDir, outputname), 'w') try: GraysonCompiler.compile (models = models, output = output, modelPath = modelPath.split (os.pathsep), namespace = namespace, version = None, logLevel = "debug", modelProperties = definitions, outputdir = tmpOutputDir, sites = sites, toLogFile = os.path.join (outputDir, "log.txt")) finally: if output: output.close () except IOError as e: logger.error ("Encountered IOError %s compiling subdax %s", e.__str__ (), output) raise e replicaText.append ('%s file://%s/%s pool="local"' % (outputname, outputDir, outputname)) template = Template (self.subdax) flowContext ['c'] = c flowContext ['outputname'] = outputname flowContext ['instanceArgs'] = instanceArgs flowContext ['sites'] = "--sites %s" % sites if instanceArgs == "" else "" text.append (template.substitute (flowContext)) replicaCatalogName = "replica-catalog.rc" masterRC = os.path.join (outputDir, replicaCatalogName) self.updateCatalog (master = masterRC, other = os.path.join (tmpOutputDir, replicaCatalogName)) c += 1 elif mapType == 'list': stream = open (inputFile, "r") c = 0 for line in stream: outputname = "%s.%s.dax" % ( outputBasename, c ) definitions = { variable : line, index : "%s" % c, Operator.DYNAMIC_INDEX : "%s" % c, "appHome" : appHome } logger.debug ("dynamic-map: invoking compiler") try: output = open (os.path.join (outputDir, outputname), 'w') try: GraysonCompiler.compile (models = models, output = output, modelPath = modelPath.split (os.pathsep), namespace = namespace, version = None, logLevel = "debug", modelProperties = definitions, outputdir = tmpOutputDir, sites = sites, toLogFile = os.path.join (outputDir, "log.txt")) finally: if output: output.close () except IOError as e: logger.error ("Encountered IOError %s compiling subdax %s", e.__str__ (), output) raise e replicaText.append ('%s file://%s/%s pool="local"' % (outputname, outputDir, outputname)) template = Template (self.subdax) flowContext ['c'] = c flowContext ['outputname'] = outputname flowContext ['instanceArgs'] = instanceArgs flowContext ['sites'] = "--sites %s" % sites if instanceArgs == "" else "" text.append (template.substitute (flowContext)) replicaCatalogName = "replica-catalog.rc" masterRC = os.path.join (outputDir, replicaCatalogName) self.updateCatalog (master = masterRC, other = os.path.join (tmpOutputDir, replicaCatalogName)) c += 1 text.append (self.footer) mainFlowContent = ''.join (text) GraysonUtil.writeFile (outputPath = os.path.join (outputDir, main_flow_name), data = mainFlowContent) logger.debug ("dynamic-map: writing output dax: %s" % mainFlowContent) replicaText.append ('%s file://%s pool="local"' % (os.path.basename (main_flow_name), main_flow_name)) GraysonUtil.writeFile (os.path.join (outputDir, "tmp", replicaCatalogName), '\n'.join (replicaText)) self.updateCatalog (master = masterRC, other = os.path.join (tmpOutputDir, replicaCatalogName)) transformationCatalogName = "transformation-catalog.tc" masterTC = os.path.join (outputDir, transformationCatalogName) self.updateCatalog (master = masterTC, other = os.path.join (tmpOutputDir, transformationCatalogName))