示例#1
0
def uploadLocalFile(job, namePattern, localDir, should_del=True):
    """
    Upload a locally available file to the grid as a DiracFile.
    Randomly chooses an SE.

    Args:
        namePattern (str): name of the file
        localDir (str): localDir of the file
        should_del = (bool): should we delete the local file?
    Return
        DiracFile: a DiracFile of the uploaded LFN on the grid
    """

    new_df = DiracFile(namePattern, localDir=localDir)
    new_df.credential_requirements=job.backend.credential_requirements
    trySEs = getConfig('DIRAC')['allDiracSE']
    random.shuffle(trySEs)
    new_lfn = os.path.join(getInputFileDir(job), namePattern)
    returnable = None
    for SE in trySEs:
        #Check that the SE is writable
        if execute('checkSEStatus("%s", "%s")' % (SE, 'Write')):
            try:
                returnable = new_df.put(force=True, uploadSE=SE, lfn=new_lfn)[0]
                break
            except GangaDiracError as err:
                raise GangaException("Upload of input file as LFN %s to SE %s failed" % (new_lfn, SE)) 
    if not returnable:
        raise GangaException("Failed to upload input file to any SE")
    if should_del:
        os.unlink(os.path.join(localDir, namePattern))

    return returnable
示例#2
0
    def upload(self, lfn, diracSE, guid=None):

        from GangaDirac.Lib.Files.DiracFile import DiracFile
        diracFile = DiracFile(namePattern=self.name, lfn=lfn)

        diracFile.put(force=True)

        return diracFile
示例#3
0
def df():
    load_config_files()

    from GangaDirac.Lib.Files.DiracFile import DiracFile
    f = DiracFile('np', 'ld', 'lfn')
    f.locations = ['location']
    f.guid = 'guid'
    yield f
    clear_config()
示例#4
0
def uploadLocalFile(job, namePattern, localDir, should_del=True):
    """
    Upload a locally available file to the grid as a DiracFile

    Args:
        namePattern (str): name of the file
        localDir (str): localDir of the file
        should_del = (bool): should we delete the local file?
    Return
        DiracFile: a DiracFile of the uploaded LFN on the grid
    """

    new_df = DiracFile(namePattern, localDir=localDir)
    random_SE = random.choice(getConfig('DIRAC')['allDiracSE'])
    new_lfn = os.path.join(getInputFileDir(job), namePattern)
    returnable = new_df.put(force=True, uploadSE=random_SE, lfn=new_lfn)[0]

    if should_del:
        os.unlink(os.path.join(localDir, namePattern))

    return returnable
示例#5
0
文件: BOOT.py 项目: MannyMoo/ganga
def getDiracFiles():
    import os
    from GangaDirac.Lib.Files.DiracFile import DiracFile
    from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
    filename = DiracFile.diracLFNBase().replace('/', '-') + '.lfns'
    logger.info('Creating list, this can take a while if you have a large number of SE files, please wait...')
    execute('dirac-dms-user-lfns &> /dev/null', shell=True, timeout=None)
    g = GangaList()
    with open(filename[1:], 'r') as lfnlist:
        lfnlist.seek(0)
        g.extend((DiracFile(lfn='%s' % lfn.strip()) for lfn in lfnlist.readlines()))
    return addProxy(g)
def generateDiracInput(app):
    """
    Construct a DIRAC input which must be unique to each job to have unique checksum.
    This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput
    Args:
        app (GaudiRun): This expects a GaudiRun app to be passed so that the constructed
    """

    input_files, input_folders = collectPreparedFiles(app)

    job = app.getJobObject()

    if input_folders:
        raise ApplicationConfigurationError(None, 'Prepared folders not supported yet, please fix this in future')
    else:
        prep_dir = app.getSharedPath()
        add_timeStampFile(prep_dir)
        prep_file = prep_dir + '.tgz'
        compressed_file = os.path.join(tempfile.gettempdir(), '__'+os.path.basename(prep_file))

        wnScript = generateWNScript(prepareCommand(app), job)
        script_name = os.path.join(tempfile.gettempdir(), wnScript.name)
        wnScript.create(script_name)

        with tarfile.open(compressed_file, "w:gz") as tar_file:
            for name in input_files:
                tar_file.add(name, arcname=os.path.basename(name))
            tar_file.add(script_name, arcname=os.path.basename(script_name))
        shutil.move(compressed_file, prep_dir)

    new_df = DiracFile(namePattern=os.path.basename(compressed_file), localDir=app.getSharedPath())
    random_SE = random.choice(getConfig('DIRAC')['allDiracSE'])
    logger.info("new File: %s" % new_df)
    new_lfn = os.path.join(DiracFile.diracLFNBase(), 'GangaInputFile/Job_%s' % job.fqid, os.path.basename(compressed_file))
    new_df.put(uploadSE=random_SE, lfn=new_lfn)

    app.uploadedInput = new_df
示例#7
0
def string_file_shortcut(v, item):
    if isinstance(v, str):
        # use proxy class to enable all user conversions on the value itself
        # but return the implementation object (not proxy)
        key = findOutputFileTypeByFileName(v)
        if key is not None:
            if key == 'MassStorageFile':
                from .MassStorageFile import MassStorageFile
                return stripProxy(MassStorageFile._proxyClass(v))
            elif key == 'LCGSEFile':
                from .LCGSEFile import LCGSEFile
                return stripProxy(LCGSEFile._proxyClass(v))
            elif key == 'DiracFile':
                try:
                    from GangaDirac.Lib.Files.DiracFile import DiracFile
                    return stripProxy(DiracFile._proxyClass(v))
                except:
                    Ganga.Utility.logging.log_unknown_exception()
                    pass

        return stripProxy(LocalFile._proxyClass(v))

    return None
示例#8
0
def getInputFileDir(job):
    """
    Return the LFN remote dirname for this job
    """
    return os.path.join(DiracFile.diracLFNBase(), 'GangaInputFile/Job_%s' % job.fqid)
示例#9
0
    def master_prepare(self, app, appmasterconfig):
        """
        Prepare the RTHandler for the master job so that applications to be submitted
        Args:
            app (GaudiExec): This application is only expected to handle GaudiExec Applications here
            appmasterconfig (unknown): Output passed from the application master configuration call
        """

        cred_req = app.getJobObject().backend.credential_requirements
        check_creds(cred_req)

        inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)

        # If we are getting the metadata we need to make sure the summary.xml is added to the output sandbox if not there already.
        if app.getMetadata and not 'summary.xml' in outputsandbox:
            outputsandbox += ['summary.xml']

        # Check a previously uploaded input is there in case of a job copy
        if isinstance(app.uploadedInput, DiracFile):
            if app.uploadedInput.getReplicas() == {}:
                app.uploadedInput = None
                logger.info("Previously uploaded cmake target missing from Dirac. Uploading it again.")

        if not isinstance(app.uploadedInput, DiracFile):
            generateDiracInput(app)
            try:
                assert isinstance(app.uploadedInput, DiracFile)
            except AssertionError:
                raise ApplicationPrepareError("Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase(cred_req))
        
        rep_data = app.uploadedInput.getReplicas()
        try:
            assert rep_data != {}
        except AssertionError:
            raise ApplicationPrepareError("Failed to find a replica of uploaded file, aborting submit")


        if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)):
            app.jobScriptArchive = None

        generateDiracScripts(app)

        try:
            assert isinstance(app.jobScriptArchive, DiracFile)
        except AssertionError:
            raise ApplicationPrepareError("Failed to upload needed file, aborting submit")
        rep_data = app.jobScriptArchive.getReplicas()
        try:
            assert rep_data != {}
        except AssertionError:
            raise ApplicationPrepareError("Failed to find a replica, aborting submit")

        #Create a replica of the job and scripts files
        replicateJobFile(app.jobScriptArchive)
        replicateJobFile(app.uploadedInput)

        return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
示例#10
0
def getInputFileDir(job):
    """
    Return the LFN remote dirname for this job
    """
    return os.path.join(DiracFile.diracLFNBase(job.backend.credential_requirements), 'GangaJob_%s/InputFiles' % job.fqid)
示例#11
0
def df():
    f = DiracFile('np', 'ld', 'lfn')
    f.locations = ['location']
    f.guid = 'guid'
    return f
示例#12
0
    def master_prepare(self, app, appmasterconfig):
        """
        Prepare the RTHandler for the master job so that applications to be submitted
        Args:
            app (GaudiExec): This application is only expected to handle GaudiExec Applications here
            appmasterconfig (unknown): Output passed from the application master configuration call
        """

        inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig)


        if not isinstance(app.uploadedInput, DiracFile):
            generateDiracInput(app)
            assert isinstance(app.uploadedInput, DiracFile), "Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase()
        
        rep_data = app.uploadedInput.getReplicas()
        assert rep_data != {}, "Failed to find a replica, aborting submit"


        if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)):
            app.jobScriptArchive = None

        generateDiracScripts(app)

        assert isinstance(app.jobScriptArchive, DiracFile), "Failed to upload needed file, aborting submit"
        rep_data = app.jobScriptArchive.getReplicas()
        assert rep_data != {}, "Failed to find a replica, aborting submit"

        return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
示例#13
0
    def getDataset(self):
        '''Gets the dataset from the bookkeeping for current path, etc.'''
        if not self.path:
            return None
        if not self.type in ['Path', 'RunsByDate', 'Run', 'Production']:
            raise GangaException('Type="%s" is not valid.' % self.type)
        if not self.type is 'RunsByDate':
            if self.startDate:
                msg = 'startDate not supported for type="%s".' % self.type
                raise GangaException(msg)
            if self.endDate:
                msg = 'endDate not supported for type="%s".' % self.type
                raise GangaException(msg)
            if self.selection:
                msg = 'selection not supported for type="%s".' % self.type
                raise GangaException(msg)
        cmd = "getDataset('%s','%s','%s','%s','%s','%s')" % (
            self.path, self.dqflag, self.type, self.startDate, self.endDate,
            self.selection)
        from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
        knownLists = [tuple, list, GangaList]
        if isType(self.dqflag, knownLists):
            cmd = "getDataset('%s',%s,'%s','%s','%s','%s')" % (
                self.path, self.dqflag, self.type, self.startDate,
                self.endDate, self.selection)
        result = get_result(cmd, 'BK query error.', 'BK query error.')

        logger.debug("Finished Running Command")

        files = []
        value = result['Value']
        if 'LFNs' in value:
            files = value['LFNs']
        if not type(files) is list:  # i.e. a dict of LFN:Metadata
            # if 'LFNs' in files: # i.e. a dict of LFN:Metadata
            files = files.keys()

        logger.debug("Creating DiracFile objects")

        ## Doesn't work not clear why
        from GangaDirac.Lib.Files.DiracFile import DiracFile
        #new_files = []
        #def _createDiracLFN(this_file):
        #    return DiracFile(lfn = this_file)
        #GangaObject.__createNewList(new_files, files, _createDiracLFN)

        new_files = [DiracFile(lfn=_file) for _file in files]

        #new_files = [DiracFile(lfn=_file) for _file in files]
        #for f in files:
        #    new_files.append(DiracFile(lfn=f))
        #ds.extend([DiracFile(lfn = f)])

        logger.debug("Creating Dataset")

        from GangaLHCb.Lib.LHCbDataset import LHCbDataset
        ds = LHCbDataset(new_files)

        logger.debug("Returning Dataset")

        return addProxy(ds)