def updateCatalog (self, masterCat, other): newResources = [] master = GraysonUtil.readFile (masterCat) masterLines = master.split ('\n') for line in masterLines: logger.debug (" __ _ _ _ _ _ _____: %s", line) sub = GraysonUtil.readFile (other) subLines = sub.split ('\n') masterMap = {} for line in masterLines: parts = line.split (' ') if len (parts) > 0: resource = parts [0] if resource: masterMap [resource] = resource logger.debug (" resource : %s", resource) for line in subLines: parts = line.split (' ') if len (parts) > 0: resource = parts [0] if resource and not resource in masterMap: newResources.append (line) logger.debug (" new resource: %s", resource) GraysonUtil.writeFile (masterCat, "%s\n%s" % (master, '\n'.join (newResources)))
def writeMetaDataCatalogs (self): GraysonUtil.writeFile ( outputPath=os.path.join (self.outputDir, PegasusProperties.SITE_CATALOG), data=self.getSiteCatalog().generateXML ()) ''' replica catalog ''' GraysonUtil.writeFile ( outputPath=os.path.join (self.outputDir, PegasusProperties.REPLICA_CATALOG), data=self.getReplicaCatalog().generateRC ()) ''' transformation catalog ''' GraysonUtil.writeFile ( outputPath=os.path.join (self.outputDir, PegasusProperties.TRANSFORMATION_CATALOG), data=self.getTransformationCatalog().generateTC ()) ''' properties ''' GraysonUtil.writeFile ( outputPath=os.path.join (os.path.join (self.outputDir, PegasusProperties.PEGASUS_PROPERTIES)), data=self.pegasusProperties.generateProperties (configDir=self.outputDir))
def put_file (request): path = request.REQUEST ["path"] content = request.REQUEST ["content"] logger.debug ("writing file: %s", path) GraysonUtil.writeFile (path, content) return ViewUtil.get_json_response ({ "status" : "ok" })
def writeFlows (self, root, flows): fileName = self.flowPath (root) workflowText = json.dumps (flows, indent=3, sort_keys=True) logger.debug (workflowText) GraysonUtil.writeFile (fileName, workflowText)
def execute (self, context={}): operatorContext = context ['operator'] method = operatorContext ["method"] inputFile = operatorContext ["input"] variable = operatorContext ["variable"] index = operatorContext ["index"] flow = operatorContext ["flow"] version = operatorContext ["version"] instanceArgs = operatorContext ["instanceArgs"] mapType = context ["mapType"] outputBasename = context ["outputName"] modelPath = context ["modelPath"] outputDir = context ["outputDir"] contextModels = context ["contextModels"] sites = context ["sites"] appHome = context ["appHome"] graysonHome = context ["graysonHome"] #print "%s" % json.dumps (context, indent=3, sort_keys=True) tmpOutputDir = os.path.join (outputDir, "tmp") # avoid overwriting replica catalog. contextModels = contextModels.split (os.pathsep) models = [ flow ] for model in contextModels: models.append (model) main_flow_name = os.path.join (outputDir, "%s.dax" % flow.replace (".graphml", "")) namespace = flow.replace (".graphml", "") flowContext = { "namespace" : outputBasename } template = Template (self.header) text = [ template.substitute (flowContext) ] replicaText = [] if mapType == 'tar': tar = tarfile.open (inputFile, "r:gz") members = tar.getmembers () c = 0 for archiveMember in members: outputname = "%s.%s.dax" % ( outputBasename, c ) definitions = { variable : archiveMember.name, index : "%s" % c, Operator.DYNAMIC_INDEX : "%s" % c, "appHome" : appHome } logger.debug ("dynamic-map: invoking compiler") try: output = open (os.path.join (outputDir, outputname), 'w') try: GraysonCompiler.compile (models = models, output = output, modelPath = modelPath.split (os.pathsep), namespace = namespace, version = None, logLevel = "debug", modelProperties = definitions, outputdir = tmpOutputDir, sites = sites, toLogFile = os.path.join (outputDir, "log.txt")) finally: if output: output.close () except IOError as e: logger.error ("Encountered IOError %s compiling subdax %s", e.__str__ (), output) raise e replicaText.append ('%s file://%s/%s pool="local"' % (outputname, outputDir, outputname)) template = Template (self.subdax) flowContext ['c'] = c flowContext ['outputname'] = outputname flowContext ['instanceArgs'] = instanceArgs flowContext ['sites'] = "--sites %s" % sites if instanceArgs == "" else "" text.append (template.substitute (flowContext)) replicaCatalogName = "replica-catalog.rc" masterRC = os.path.join (outputDir, replicaCatalogName) self.updateCatalog (master = masterRC, other = os.path.join (tmpOutputDir, replicaCatalogName)) c += 1 elif mapType == 'list': stream = open (inputFile, "r") c = 0 for line in stream: outputname = "%s.%s.dax" % ( outputBasename, c ) definitions = { variable : line, index : "%s" % c, Operator.DYNAMIC_INDEX : "%s" % c, "appHome" : appHome } logger.debug ("dynamic-map: invoking compiler") try: output = open (os.path.join (outputDir, outputname), 'w') try: GraysonCompiler.compile (models = models, output = output, modelPath = modelPath.split (os.pathsep), namespace = namespace, version = None, logLevel = "debug", modelProperties = definitions, outputdir = tmpOutputDir, sites = sites, toLogFile = os.path.join (outputDir, "log.txt")) finally: if output: output.close () except IOError as e: logger.error ("Encountered IOError %s compiling subdax %s", e.__str__ (), output) raise e replicaText.append ('%s file://%s/%s pool="local"' % (outputname, outputDir, outputname)) template = Template (self.subdax) flowContext ['c'] = c flowContext ['outputname'] = outputname flowContext ['instanceArgs'] = instanceArgs flowContext ['sites'] = "--sites %s" % sites if instanceArgs == "" else "" text.append (template.substitute (flowContext)) replicaCatalogName = "replica-catalog.rc" masterRC = os.path.join (outputDir, replicaCatalogName) self.updateCatalog (master = masterRC, other = os.path.join (tmpOutputDir, replicaCatalogName)) c += 1 text.append (self.footer) mainFlowContent = ''.join (text) GraysonUtil.writeFile (outputPath = os.path.join (outputDir, main_flow_name), data = mainFlowContent) logger.debug ("dynamic-map: writing output dax: %s" % mainFlowContent) replicaText.append ('%s file://%s pool="local"' % (os.path.basename (main_flow_name), main_flow_name)) GraysonUtil.writeFile (os.path.join (outputDir, "tmp", replicaCatalogName), '\n'.join (replicaText)) self.updateCatalog (master = masterRC, other = os.path.join (tmpOutputDir, replicaCatalogName)) transformationCatalogName = "transformation-catalog.tc" masterTC = os.path.join (outputDir, transformationCatalogName) self.updateCatalog (master = masterTC, other = os.path.join (tmpOutputDir, transformationCatalogName))
def updateCatalog (self, master, other): masterText = GraysonUtil.readFile (master) text = GraysonUtil.readFile (other) GraysonUtil.writeFile (master, "%s\n%s" % (masterText, text))