Example #1
0
    def generateTFC(self):
        """
        _generateTFC_

        Method to generate on the fly TFC. Each LFN would be mapped to local PFN 
        """ 
        workingDir = os.path.join(os.getcwd(),'prestage')
        if not os.path.exists(workingDir):
           os.mkdirs(workingDir)
        
        tfcFile = IMProvDoc ('storage-mapping')

        for item in self.localFiles.keys():

           if item.startswith('/'):
              temp = item.split('/',1)[1]

           temp = os.path.split(temp) 

           params = {'protocol':'local-stage-in','path-match':'/%s/(%s)' % (temp[0],temp[1]),'result':'file:%s/$1' % workingDir}

           node = IMProvNode('lfn-to-pfn',None,**params)
           tfcFile.addNode(node)
        
        handle = open('%s/prestageTFC.xml' % workingDir, 'w')
        handle.write (tfcFile.makeDOMDocument().toprettyxml() )
        handle.close()           
       
        return
Example #2
0
    def write(self, filename):
        """
        _write_

        Write out an XML file containing all of the RunResDB nodes
        currently in memory, removing any URL references embedded in them
        to avoid loading duplicates when the file produced is read

        """
        masterNode = IMProvDoc("RunResOutput")
        nodelist = []
        for key in self.keys():
            if self[key] == None:
                continue
            nodelist.extend(self[key])

            
        for node in nodelist:
            if "RunResDBURL" in node.keys():
                del node["RunResDBURL"]
                for i in range(0, len(node.children)):
                    item = node.children[i]
                    if item.name == "RunResDBURL":
                        node.children.pop(i)
                        
            masterNode.addNode(node)
            
        handle = open(filename, 'w')
        handle.write(masterNode.makeDOMDocument().toprettyxml())
        handle.close()
        return
Example #3
0
    def save(self, filename):
        """
        _save_
        
        Write this Configuration out as an XML file to the
        file provided by the argument

        Args --

        - *filename* : Path where file will be created. Will overwrite any
        existing file
        
        """
        #  //
        # // Serialise the Task Tree
        #//
        if self._TaskTree == None:
            msg = "Error: Task Tree is empty\n"
            msg += "You must provide a set of ShREEKTasks to execute!\n"
            raise ShREEKException(msg, ClassInstance = self)
        
        taskNode = IMProvNode("ShREEKTaskTree")
        self.addNode(taskNode)
        taskNode.addNode(self._TaskTree.makeIMProv())
        
        #  //
        # // Save Config to file as XML Document
        #//
        doc = IMProvDoc("ShREEKConfiguration")
        doc.addNode(self)
        handle = open(filename, "w")
        handle.write(doc.makeDOMDocument().toprettyxml())
        handle.close()
        return
Example #4
0
    def saveToFile(self, filename):
        """
        _saveToFile_

        Save this instance to the file provided

        """
        doc = IMProvDoc("ProdCommonConfig")
        doc.addNode(self.save())
        handle = open(filename, 'w')
        handle.write(doc.makeDOMDocument().toprettyxml())
        handle.close()
        return
Example #5
0
    def writeToFile(self, filename):
        """
        _writeToFile_

        Write data in this instance to file specified as XML

        """
        doc = IMProvDoc("ProdAgentPluginConfig")
        doc.addNode(self.save())
        handle = open(filename, 'w')
        handle.write(doc.makeDOMDocument().toprettyxml())
        handle.close()
        return
Example #6
0
    def saveToFile(self, filename):
        """
        _saveToFile_

        Save this instance to the file provided

        """
        doc = IMProvDoc("ProdAgentConfig")
        doc.addNode(self.save())
        handle = open(filename, 'w')
        handle.write(doc.makeDOMDocument().toprettyxml())
        handle.close()
        return
Example #7
0
def writeSpecFile(filename, *specInstances):
    """
    _writeSpecFile_

    Util to write multiple specs to a single file

    """
    doc = IMProvDoc("RequestSpecs")
    for spec in specInstances:
        doc.addNode(spec.save())

    handle = open(filename, 'w')
    handle.write(doc.makeDOMDocument().toprettyxml())
    handle.close()
    return
Example #8
0
def writeSpecFile(filename, *tests):
    """
    _writeSpecFile_

    Gather a list of tests into a RelVal Spec file suitable for
    injection into a PA

    """
    doc = IMProvDoc("RelValSpec")
    for test in tests:
        doc.addNode(test.save())
    handle = open(filename, 'w')
    handle.write(doc.makeDOMDocument().toprettyxml())
    handle.close()
    return
Example #9
0
def writeSpec(filename, *relValInstances):
    """
    _writeSpec_

    Write out the RelValidation instances provided to the filename
    given. Will overwrite file if present

    """
    masterDoc = IMProvDoc("ReleaseValidation")
    for item in relValInstances:
        masterDoc.addNode(item.save())
    handle = open(filename, 'w')
    handle.write(masterDoc.makeDOMDocument().toprettyxml())
    handle.close()
    return
Example #10
0
def writeSpecFile(filename, *specInstances):
    """
    _writeSpecFile_

    Util to write multiple specs to a single file

    """
    doc = IMProvDoc("RequestSpecs")
    for spec in specInstances:
        doc.addNode(spec.save())

    handle = open(filename, 'w')
    handle.write(doc.makeDOMDocument().toprettyxml())
    handle.close()
    return
Example #11
0
    def merge_reports(self):
        reports = []
        tdir = os.path.dirname(self.__jobdir)
        for path in self.__rpaths:
            f = gzip.open(os.path.join(os.path.dirname(tdir), path, 'report.xml.gz'))
            report = readJobReport(f)
            reports.extend(report)
            f.close()

        output = IMProvDoc("JobReports")
        for item in reports:
            output.addNode(item.save())

        outfile = gzip.open(os.path.join(self.__jobdir, 'report.xml.gz'), 'wb')
        outfile.write(output.makeDOMDocument().toprettyxml())
        outfile.close()
Example #12
0
def updateReport(reportFile, newReportInstance):
    """
    _updateReport_

    Given a file containing several reports: reportFile,
    find the report in there whose name matches the newReportInstance's
    name and replace that report with the new Report instance.

    Returns a boolean: True if report name was matched and updated,
    False if the report was not found and updated. (False may indicate that
    the new report file needs to be merged with the main report file)

    """
    if not os.path.exists(reportFile):
        existingReports = []
    else:
        existingReports = readJobReport(reportFile)

    updatedReport = False
    output = IMProvDoc("JobReports")
    for report in existingReports:
        if report.name == newReportInstance.name:
            output.addNode(newReportInstance.save())
            updatedReport = True
        else:
            output.addNode(report.save())

    handle = open(reportFile, 'w')
    handle.write(output.makeDOMDocument().toprettyxml())
    handle.close()
    return updatedReport
Example #13
0
    def writeMainDB(self, targetFile):
        """
        _writeMainDB_

        Write the main RunResDB file in the location provided
        so that all the components can be welded into a single RunResDB
        structure

        """
        db = IMProvDoc("RunResDB")
        for url in self.runresFiles:
            entry = IMProvNode("RunResDBURL", None, URL = url)
            db.addNode(entry)

        handle = open(targetFile, 'w')
        handle.write(db.makeDOMDocument().toprettyxml())
        handle.close()
        return
Example #14
0
    def save(self):
        """
        this object -> improvNode

        """
        result = IMProvDoc("PileupDataset")
        node = IMProvNode("Dataset", None,
                          Name = self.dataset,
                          FilesPerJob = str(self.filesPerJob))
        if self.overlap:
            node.attrs['Overlap'] = "True"
        else:
            node.attrs['Overlap'] = "False"
        
        result.addNode(node)
        for lfn in self:
            node.addNode(IMProvNode("LFN", lfn))
        return result
Example #15
0
    def save(self, directory):
        """
        _save_

        Save details of this object to the dir provided using
        the basename of the workflow file

        """
        doc = IMProvDoc("DatasetIterator")
        node = IMProvNode(self.workflowSpec.workflowName())
        doc.addNode(node)

        node.addNode(IMProvNode("Run", None, Value = str(self.count)))

        node.addNode(IMProvNode("SplitType", None, Value = str(self.splitType)))
        node.addNode(IMProvNode("SplitSize", None, Value = str(self.splitSize)))

        

        pu = IMProvNode("Pileup")
        node.addNode(pu)
        for key, value in self.pileupDatasets.items():
            puNode = value.save()
            puNode.attrs['PayloadNode'] = key
            pu.addNode(puNode)

        specs = IMProvNode("JobSpecs")
        node.addNode(specs)
        for key, val in self.ownedJobSpecs.items():
            specs.addNode(IMProvNode("JobSpec", val, ID = key))
            
        fname = os.path.join(
            directory,
            "%s-Persist.xml" % self.workflowSpec.workflowName()
            )
        handle = open(fname, 'w')
        handle.write(doc.makeDOMDocument().toprettyxml())
        handle.close()
        
        
        return
Example #16
0
def updateReport(reportFile, newReportInstance):
    """
    _updateReport_

    Given a file containing several reports: reportFile,
    find the report in there whose name matches the newReportInstance's
    name and replace that report with the new Report instance.

    Returns a boolean: True if report name was matched and updated,
    False if the report was not found and updated. (False may indicate that
    the new report file needs to be merged with the main report file)

    """
    if not os.path.exists(reportFile):
        existingReports = []
    else:
        existingReports = readJobReport(reportFile)

    updatedReport = False
    output = IMProvDoc("JobReports")
    for report in existingReports:
        if report.name == newReportInstance.name:
            output.addNode(newReportInstance.save())
            updatedReport = True
        else:
            output.addNode(report.save())

    
    handle = open(reportFile, 'w')
    handle.write(output.makeDOMDocument().toprettyxml())
    handle.close()
    return updatedReport
Example #17
0
    def save(self, directory):
        """
        _save_

        Persist this objects state into an XML file and save
        it in the directory provided

        """
        doc = IMProvDoc("RequestIterator")
        node = IMProvNode(self.workflowSpec.workflowName())
        doc.addNode(node)

        node.addNode(IMProvNode("Run", None, Value = str(self.count)))
        node.addNode(
            IMProvNode("EventsPerJob", None, Value = str(self.eventsPerJob))
            )
        node.addNode(IMProvNode("SitePref", None, Value = str(self.sitePref)))

        pu = IMProvNode("Pileup")
        node.addNode(pu)
        for key, value in self.pileupDatasets.items():
            puNode = value.save()
            puNode.attrs['PayloadNode'] = key
            pu.addNode(puNode)

        specs = IMProvNode("JobSpecs")
        node.addNode(specs)
        for key, val in self.ownedJobSpecs.items():
            specs.addNode(IMProvNode("JobSpec", val, ID = key))
            
        fname = os.path.join(
            directory,
            "%s-Persist.xml" % self.workflowSpec.workflowName()
            )
        handle = open(fname, 'w')
        handle.write(doc.makeDOMDocument().toprettyxml())
        handle.close()
        return
Example #18
0
def mergeReports(reportFile1, reportFile2):
    """
    _mergeReports_

    Load job reports from both files, and combine them into a
    single file.

    The output will be written to the first file provided.
    (IE JobReports from reportFile2 will be added to reportFile1)

    If reportFile1 does not exist, a new report will be created, containing
    the contents of reportFile2.
    
    If reportFile2 does not exist, then a RuntimeError is thrown.

    """
    if not os.path.exists(reportFile1):
        reports1 = []
    else:
        reports1 = readJobReport(reportFile1)

    if not os.path.exists(reportFile2):
        msg = "Report file to be merged does not exist:\n"
        msg += reportFile2
        raise RuntimeError, msg

    reports2 = readJobReport(reportFile2)
    reports1.extend(reports2)

    
    output = IMProvDoc("JobReports")
    for item in reports1:
        output.addNode(item.save())
    handle = open(reportFile1, 'w')
    handle.write(output.makeDOMDocument().toprettyxml())
    handle.close()
    return
Example #19
0
def mergeReports(reportFile1, reportFile2):
    """
    _mergeReports_

    Load job reports from both files, and combine them into a
    single file.

    The output will be written to the first file provided.
    (IE JobReports from reportFile2 will be added to reportFile1)

    If reportFile1 does not exist, a new report will be created, containing
    the contents of reportFile2.
    
    If reportFile2 does not exist, then a RuntimeError is thrown.

    """
    if not os.path.exists(reportFile1):
        reports1 = []
    else:
        reports1 = readJobReport(reportFile1)

    if not os.path.exists(reportFile2):
        msg = "Report file to be merged does not exist:\n"
        msg += reportFile2
        raise RuntimeError, msg

    reports2 = readJobReport(reportFile2)
    reports1.extend(reports2)

    output = IMProvDoc("JobReports")
    for item in reports1:
        output.addNode(item.save())
    handle = open(reportFile1, 'w')
    handle.write(output.makeDOMDocument().toprettyxml())
    handle.close()
    return
Example #20
0
    def save(self):
        """
        _save_

        Convert this object into an IMProvNode structure

        """
        result = IMProvDoc("RequestSpec")

        details = IMProvNode("RequestDetails")
        for key, val in self.requestDetails.items():
            details.addNode(IMProvNode(key, str(val)))

        policies = IMProvNode("Policies")
        for key, val in self.policies.items():
            policies.addNode(IMProvNode(key, str(val)))

        result.addNode(details)
        result.addNode(policies)
        result.addNode(self.workflow.makeIMProv())
        return result
Example #21
0
    def startElement(self, name, attrs):
        """
        _startElement_

        Override SAX startElement handler
        """
        plainAttrs = {}
        for key, value in attrs.items():
            plainAttrs[str(key)] = str(value)
        if self._ParentDoc == None:
            self._ParentDoc = IMProvDoc(str(name))
            self._ParentDoc.attrs.update(plainAttrs)
            self._NodeStack.append(self._ParentDoc)
            return

        self._CharCache = []
        newnode = IMProvNode(str(name))
        for key, value in attrs.items():
            newnode.attrs[key] = value
        self._NodeStack[-1].addNode(newnode)
        self._NodeStack.append(newnode)
        return
Example #22
0
    def save(self):
        """
        _save_

        Convert this object into an IMProvNode structure

        """
        result = IMProvDoc("RequestSpec")


        details = IMProvNode("RequestDetails")
        for key, val in self.requestDetails.items():
            details.addNode(IMProvNode(key, str(val)))

        policies = IMProvNode("Policies")
        for key, val in self.policies.items():
            policies.addNode(IMProvNode(key, str(val)))
            
            
        result.addNode(details)
        result.addNode(policies)
        result.addNode(self.workflow.makeIMProv())
        return result
Example #23
0
            # longer a session leader, preventing the daemon from ever acquiring
            # a controlling terminal.
            pid = os.fork()  # Fork a second child.
        except OSError, e:
            raise Exception, "%s [%d]" % (e.strerror, e.errno)

        if (pid == 0):  # The second child.
            # Since the current working directory may be a mounted filesystem, we
            # avoid the issue of not being able to unmount the filesystem at
            # shutdown time by changing it to the root directory.
            os.chdir(workdir)
            # We probably don't want the file mode creation mask inherited from
            # the parent, so we give the child complete control over permissions.
            os.umask(UMASK)

            doc = IMProvDoc("Daemon")
            doc.addNode(IMProvNode("ProcessID", None, Value=os.getpid()))
            doc.addNode(IMProvNode("ParentProcessID", None,
                                   Value=os.getppid()))
            doc.addNode(IMProvNode("ProcessGroupID", None, Value=os.getpgrp()))
            doc.addNode(IMProvNode("UserID", None, Value=os.getuid()))
            doc.addNode(IMProvNode("EffectiveUserID", None,
                                   Value=os.geteuid()))
            doc.addNode(IMProvNode("GroupID", None, Value=os.getgid()))
            doc.addNode(
                IMProvNode("EffectiveGroupID", None, Value=os.getegid()))

            open("Daemon.xml", "w").write(doc.makeDOMDocument().toprettyxml())
            print "Started Daemon: Process %s" % os.getpid()

        else:
Example #24
0
def combineReports(reportFile, reportNames, newReportInstance):
    """
    Combine reports, take some fields from report with given name
    in reportFile and then overwrite with newReportInstance
    
    Note: newReportInstance is modified, and should be written back as the 
    task fjr - else subsequent tasks will take the wrong one!!!
    
    """
    if not os.path.exists(reportFile):
        existingReports = []
    else:
        existingReports = readJobReport(reportFile)

    if not isinstance(reportNames, list):
        reportNames = [reportNames]

    reportFound = False
    output = IMProvDoc("JobReports")

    #wipe old values ready for new ones
    newReportInstance.inputFiles = []
    newReportInstance.generatorInfo = {}  #how to handle multiple?

    for report in existingReports:
        if report.name in reportNames:
            reportFound = True

            # copy some values across from old report
            newReportInstance.inputFiles.extend(report.inputFiles)
            newReportInstance.skippedEvents.extend(report.skippedEvents)
            newReportInstance.skippedFiles.extend(report.skippedFiles)

            # loop over output files and change provenance to 1st node's
            for outfile in newReportInstance.files:
                oldinputfiles = outfile.inputFiles
                outfile.inputFiles = [
                ]  #clear ready for correct provenance info
                for infile in oldinputfiles:
                    # find the ancestor input files in previous report
                    for ancestor in report.files:
                        if ancestor['LFN'] == infile['LFN']:
                            outfile.inputFiles.extend(ancestor.inputFiles)
                            print "Updated InputFiles %s for %s" % (
                                ancestor.inputFiles, outfile['LFN'])
                        # No LFN, use PFN (Needed for parent forwarding)
                        elif not infile['LFN'] and \
                                ancestor['PFN'] == infile['PFN']:
                            outfile.inputFiles.extend(ancestor.inputFiles)
                            print "Updated InputFiles %s for %s" % (
                                ancestor.inputFiles, outfile['LFN'])

            if report.timing.has_key('AppStartTime') and \
                report.timing['AppStartTime'] < newReportInstance.timing.get('AppStartTime', time.time()):
                newReportInstance.timing['AppStartTime'] = report.timing[
                    'AppStartTime']
            continue

        #  // if here either this report is not one of the inputs
        # //     or the report contained a staged out file
        #//            - in either case it must be saved
        output.addNode(report.save())

    if not reportFound:
        raise RuntimeError, "Reports not combined: %s not found in %s" % \
                        (str(reportNames), reportFile)

    output.addNode(newReportInstance.save())

    handle = open(reportFile, 'w')
    handle.write(output.makeDOMDocument().toprettyxml())
    handle.close()
    return newReportInstance
Example #25
0
         # longer a session leader, preventing the daemon from ever acquiring
         # a controlling terminal.
         pid = os.fork()	# Fork a second child.
      except OSError, e:
         raise Exception, "%s [%d]" % (e.strerror, e.errno)

      if (pid == 0):	# The second child.
         # Since the current working directory may be a mounted filesystem, we
         # avoid the issue of not being able to unmount the filesystem at
         # shutdown time by changing it to the root directory.
         os.chdir(workdir)
         # We probably don't want the file mode creation mask inherited from
         # the parent, so we give the child complete control over permissions.
         os.umask(UMASK)

         doc = IMProvDoc("Daemon")
         doc.addNode(IMProvNode("ProcessID", None, Value = os.getpid()))
         doc.addNode(IMProvNode("ParentProcessID", None, Value = os.getppid()))
         doc.addNode(IMProvNode("ProcessGroupID", None, Value = os.getpgrp()))
         doc.addNode(IMProvNode("UserID", None, Value = os.getuid()))
         doc.addNode(IMProvNode("EffectiveUserID", None, Value = os.geteuid()))
         doc.addNode(IMProvNode("GroupID", None, Value = os.getgid()))
         doc.addNode(IMProvNode("EffectiveGroupID", None,
                                Value = os.getegid()))
         
         open("Daemon.xml", "w").write(doc.makeDOMDocument().toprettyxml())
         print "Started Daemon: Process %s" % os.getpid()
         
      else:
         # exit() or _exit()?  See below.
         os._exit(0)	# Exit parent (the first child) of the second child.
Example #26
0
def combineReports(reportFile, reportNames, newReportInstance):
    """
    Combine reports, take some fields from report with given name
    in reportFile and then overwrite with newReportInstance
    
    Note: newReportInstance is modified, and should be written back as the 
    task fjr - else subsequent tasks will take the wrong one!!!
    
    """
    if not os.path.exists(reportFile):
        existingReports = []
    else:
        existingReports = readJobReport(reportFile)

    if not isinstance(reportNames, list):
        reportNames = [reportNames]

    reportFound = False
    output = IMProvDoc("JobReports")
    
    #wipe old values ready for new ones
    newReportInstance.inputFiles = []
    newReportInstance.generatorInfo = {} #how to handle multiple?
    
    for report in existingReports:
        if report.name in reportNames:
            reportFound = True
            
            # copy some values across from old report
            newReportInstance.inputFiles.extend(report.inputFiles)
            newReportInstance.skippedEvents.extend(report.skippedEvents)
            newReportInstance.skippedFiles.extend(report.skippedFiles)
    
            # loop over output files and change provenance to 1st node's
            for outfile in newReportInstance.files:
                oldinputfiles = outfile.inputFiles
                outfile.inputFiles = [] #clear ready for correct provenance info
                for infile in oldinputfiles:
                    # find the ancestor input files in previous report
                    for ancestor in report.files:
                        if ancestor['LFN'] == infile['LFN']:
                            outfile.inputFiles.extend(ancestor.inputFiles)
                            print "Updated InputFiles %s for %s" % (
                                ancestor.inputFiles, outfile['LFN'])
                        # No LFN, use PFN (Needed for parent forwarding)
                        elif not infile['LFN'] and \
                                ancestor['PFN'] == infile['PFN']:
                            outfile.inputFiles.extend(ancestor.inputFiles)
                            print "Updated InputFiles %s for %s" % (
                                ancestor.inputFiles, outfile['LFN'])
                       
            if report.timing.has_key('AppStartTime') and \
                report.timing['AppStartTime'] < newReportInstance.timing.get('AppStartTime', time.time()):
                newReportInstance.timing['AppStartTime'] = report.timing['AppStartTime']
            continue
        
        #  // if here either this report is not one of the inputs
        # //     or the report contained a staged out file
        #//            - in either case it must be saved
        output.addNode(report.save())
    
    if not reportFound:
        raise RuntimeError, "Reports not combined: %s not found in %s" % \
                        (str(reportNames), reportFile)
    
    output.addNode(newReportInstance.save())
    
    handle = open(reportFile, 'w')
    handle.write(output.makeDOMDocument().toprettyxml())
    handle.close()
    return newReportInstance