Пример #1
0
    def upload(self, lfn, diracSE, guid=None):

        from GangaDirac.Lib.Files.DiracFile import DiracFile
        diracFile = DiracFile(namePattern=self.name, lfn=lfn)

        diracFile.put(force=True)

        return diracFile
Пример #2
0
    def upload(self, lfn, diracSE, guid=None):

        from GangaDirac.Lib.Files.DiracFile import DiracFile
        diracFile = DiracFile(namePattern=self.name, lfn=lfn)

        diracFile.put(force=True)

        return diracFile
Пример #3
0
def uploadLocalFile(job, namePattern, localDir, should_del=True):
    """
    Upload a locally available file to the grid as a DiracFile.
    Randomly chooses an SE.

    Args:
        namePattern (str): name of the file
        localDir (str): localDir of the file
        should_del = (bool): should we delete the local file?
    Return
        DiracFile: a DiracFile of the uploaded LFN on the grid
    """

    new_df = DiracFile(namePattern, localDir=localDir)
    new_df.credential_requirements=job.backend.credential_requirements
    trySEs = getConfig('DIRAC')['allDiracSE']
    random.shuffle(trySEs)
    new_lfn = os.path.join(getInputFileDir(job), namePattern)
    returnable = None
    for SE in trySEs:
        #Check that the SE is writable
        if execute('checkSEStatus("%s", "%s")' % (SE, 'Write')):
            try:
                returnable = new_df.put(force=True, uploadSE=SE, lfn=new_lfn)[0]
                break
            except GangaDiracError as err:
                raise GangaException("Upload of input file as LFN %s to SE %s failed" % (new_lfn, SE)) 
    if not returnable:
        raise GangaException("Failed to upload input file to any SE")
    if should_del:
        os.unlink(os.path.join(localDir, namePattern))

    return returnable
Пример #4
0
def uploadLocalFile(job, namePattern, localDir, should_del=True):
    """
    Upload a locally available file to the grid as a DiracFile.
    Randomly chooses an SE.

    Args:
        namePattern (str): name of the file
        localDir (str): localDir of the file
        should_del = (bool): should we delete the local file?
    Return
        DiracFile: a DiracFile of the uploaded LFN on the grid
    """

    new_df = DiracFile(namePattern, localDir=localDir)
    trySEs = getConfig('DIRAC')['allDiracSE']
    random.shuffle(trySEs)
    new_lfn = os.path.join(getInputFileDir(job), namePattern)
    returnable = None
    for SE in trySEs:
        #Check that the SE is writable
        if execute('checkSEStatus("%s", "%s")' % (SE, 'Write')):
            try:
                returnable = new_df.put(force=True, uploadSE=SE, lfn=new_lfn)[0]
                break
            except GangaDiracError as err:
                raise GangaException("Upload of input file as LFN %s to SE %s failed" % (new_lfn, SE)) 
    if not returnable:
        raise GangaException("Failed to upload input file to any SE")
    if should_del:
        os.unlink(os.path.join(localDir, namePattern))

    return returnable
def generateDiracInput(app):
    """
    Construct a DIRAC input which must be unique to each job to have unique checksum.
    This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput
    Args:
        app (GaudiRun): This expects a GaudiRun app to be passed so that the constructed
    """

    input_files, input_folders = collectPreparedFiles(app)

    job = app.getJobObject()

    if input_folders:
        raise ApplicationConfigurationError(None, 'Prepared folders not supported yet, please fix this in future')
    else:
        prep_dir = app.getSharedPath()
        add_timeStampFile(prep_dir)
        prep_file = prep_dir + '.tgz'
        compressed_file = os.path.join(tempfile.gettempdir(), '__'+os.path.basename(prep_file))

        wnScript = generateWNScript(prepareCommand(app), job)
        script_name = os.path.join(tempfile.gettempdir(), wnScript.name)
        wnScript.create(script_name)

        with tarfile.open(compressed_file, "w:gz") as tar_file:
            for name in input_files:
                tar_file.add(name, arcname=os.path.basename(name))
            tar_file.add(script_name, arcname=os.path.basename(script_name))
        shutil.move(compressed_file, prep_dir)

    new_df = DiracFile(namePattern=os.path.basename(compressed_file), localDir=app.getSharedPath())
    random_SE = random.choice(getConfig('DIRAC')['allDiracSE'])
    logger.info("new File: %s" % new_df)
    new_lfn = os.path.join(DiracFile.diracLFNBase(), 'GangaInputFile/Job_%s' % job.fqid, os.path.basename(compressed_file))
    new_df.put(uploadSE=random_SE, lfn=new_lfn)

    app.uploadedInput = new_df
Пример #6
0
def uploadLocalFile(job, namePattern, localDir, should_del=True):
    """
    Upload a locally available file to the grid as a DiracFile

    Args:
        namePattern (str): name of the file
        localDir (str): localDir of the file
        should_del = (bool): should we delete the local file?
    Return
        DiracFile: a DiracFile of the uploaded LFN on the grid
    """

    new_df = DiracFile(namePattern, localDir=localDir)
    random_SE = random.choice(getConfig('DIRAC')['allDiracSE'])
    new_lfn = os.path.join(getInputFileDir(job), namePattern)
    returnable = new_df.put(force=True, uploadSE=random_SE, lfn=new_lfn)[0]

    if should_del:
        os.unlink(os.path.join(localDir, namePattern))

    return returnable
Пример #7
0
def uploadLocalFile(job, namePattern, localDir, should_del=True):
    """
    Upload a locally available file to the grid as a DiracFile

    Args:
        namePattern (str): name of the file
        localDir (str): localDir of the file
        should_del = (bool): should we delete the local file?
    Return
        DiracFile: a DiracFile of the uploaded LFN on the grid
    """

    new_df = DiracFile(namePattern, localDir=localDir)
    random_SE = random.choice(getConfig('DIRAC')['allDiracSE'])
    new_lfn = os.path.join(getInputFileDir(job), namePattern)
    returnable = new_df.put(force=True, uploadSE=random_SE, lfn=new_lfn)[0]

    if should_del:
        os.unlink(os.path.join(localDir, namePattern))

    return returnable