Пример #1
0
def AlienCopy(source, destination, attempts=3, overwrite=False):
    i = 0
    fileExists = False

    if AlienFileExists(destination):
        if overwrite:
            AlienDelete(destination)
        else:
            return True

    if destination.find("alien://") == -1:
        dest = "alien://{0}".format(destination)
    else:
        dest = destination

    while True:
        alisimtools.subprocess_call(["alien_cp", source, dest])
        i += 1
        fileExists = AlienFileExists(destination)
        if fileExists:
            break
        if i >= attempts:
            logging.info(
                "After {0} attempts I could not copy {1} to {2}".format(
                    i, source, dest))
            break

    return fileExists
Пример #2
0
def CopyFilesToTheGrid(Files, AlienDest, LocalDest, Offline, GridUpdate):
    if not Offline:
        alisimtools.subprocess_call(["alien_mkdir", "-p", AlienDest])
        alisimtools.subprocess_call(
            ["alien_mkdir", "-p", "{0}/output".format(AlienDest)])

    if not os.path.isdir(LocalDest):
        logging.info("Creating directory %s", LocalDest)
        os.makedirs(LocalDest)
    for file in Files:
        fname = os.path.basename(file)
        if not Offline:
            AlienCopy(file, "alien://{}/{}".format(AlienDest, fname), 3,
                      GridUpdate)
        shutil.copy(file, os.path.join(LocalDest, fname))
Пример #3
0
def DownloadResults(TrainName, LocalPath, AlienPath, Gen, Proc, PtHardList,
                    MergingStage):
    if PtHardList and len(PtHardList) > 1:
        minPtHardBin = 0
        maxPtHardBin = len(PtHardList) - 1
    else:
        minPtHardBin = -1
        maxPtHardBin = 0

    for ptHardBin in range(minPtHardBin, maxPtHardBin):
        if ptHardBin < 0:
            TrainPtHardName = TrainName
        else:
            minPtHard = PtHardList[ptHardBin]
            maxPtHard = PtHardList[ptHardBin + 1]
            TrainPtHardName = "{0}/{1}".format(TrainName, ptHardBin)
        logging.info(
            "Downloading results from train {0}".format(TrainPtHardName))
        if MergingStage < 0:
            MergingStage = DetermineMergingStage(AlienPath, TrainPtHardName)

        if MergingStage < 0:
            logging.error(
                "Could not find any results from train {0}! Aborting...".
                format(TrainPtHardName))
            exit(0)
        elif MergingStage == 0:
            logging.warning(
                "Merging stage determined to be 0 (i.e. no grid merging has been performed)"
            )
            AlienOutputPath = "{0}/{1}/output".format(AlienPath,
                                                      TrainPtHardName)
            LocalDest = "{0}/{1}/output".format(LocalPath, TrainPtHardName)
        else:
            logging.info(
                "Merging stage determined to be {0}".format(MergingStage))
            AlienOutputPath = "{0}/{1}/stage_{2}/output".format(
                AlienPath, TrainPtHardName, MergingStage - 1)
            LocalDest = "{0}/{1}/stage_{2}/output".format(
                LocalPath, TrainPtHardName, MergingStage - 1)

        if not os.path.isdir(LocalDest):
            os.makedirs(LocalDest)
        AlienOuputContent = alisimtools.subprocess_checkoutput(
            ["alien_ls", AlienOutputPath]).splitlines()
        for SubDir in AlienOuputContent:
            i = SubDir.rfind("/")
            if i >= 0: SubDir = SubDir[i + 1:]
            SubDirDest = "{0}/{1}".format(LocalDest, SubDir)
            SubDirOrig = "{0}/{1}".format(AlienOutputPath, SubDir)
            if not os.path.isdir(SubDirDest):
                os.makedirs(SubDirDest)
            FilesToDownload = alisimtools.subprocess_checkoutput(
                ["alien_ls",
                 "{0}/AnalysisResults*.root".format(SubDirOrig)]).splitlines()
            for FileName in FilesToDownload:
                i = FileName.rfind("/")
                if i >= 0: FileName = FileName[i + 1:]
                FileDest = "{0}/{1}".format(SubDirDest, FileName)
                if os.path.isfile(FileDest):
                    logging.warning(
                        "File {0} already exists, skipping...".format(
                            FileDest))
                    continue
                FileOrig = "{0}/{1}".format(SubDirOrig, FileName)
                FileDestTemp = "{0}/temp_{1}".format(SubDirDest, FileName)
                if os.path.isfile(FileDestTemp):
                    os.remove(FileDestTemp)
                logging.info("Downloading from {0} to {1}".format(
                    FileOrig, FileDestTemp))
                alisimtools.subprocess_call(
                    ["alien_cp", "alien://{0}".format(FileOrig), FileDestTemp])
                if os.path.getsize(FileDestTemp) > 0:
                    logging.info("Renaming {0} to {1}".format(
                        FileDestTemp, FileDest))
                    os.rename(FileDestTemp, FileDest)
                else:
                    logging.error(
                        "Downloading of {0} failed!".format(FileOrig))
                    os.remove(FileDestTemp)
Пример #4
0
def SubmitProcessingJobs(TrainName, LocalPath, AlienPath, AliPhysicsVersion,
                         Offline, GridUpdate, TTL, Events, Jobs, Gen, Proc,
                         yamlFileName, PtHardList, OldPowhegInit, PowhegStage,
                         HerwigTune, LoadPackagesSeparately):
    logging.info("Submitting processing jobs for train {0}".format(TrainName))

    ValidationScript = "FastSim_validation.sh"
    ExeFile = "runFastSim.py"
    JdlFile = "FastSim_{0}_{1}.jdl".format(Gen, Proc)

    FilesToDelete = [JdlFile]

    FilesToCopy = [
        yamlFileName, "OnTheFlySimulationGenerator.cxx",
        "OnTheFlySimulationGenerator.h", "runJetSimulation.C",
        "start_simulation.C", "lhapdf_utils.py", "Makefile", "HepMC.tar",
        "AliGenExtFile_dev.h", "AliGenExtFile_dev.cxx",
        "AliGenReaderHepMC_dev.h", "AliGenReaderHepMC_dev.cxx",
        "AliGenEvtGen_dev.h", "AliGenEvtGen_dev.cxx", "AliGenPythia_dev.h",
        "AliGenPythia_dev.cxx", "AliPythia6_dev.h", "AliPythia6_dev.cxx",
        "AliPythia8_dev.h", "AliPythia8_dev.cxx", "AliPythiaBase_dev.h",
        "AliPythiaBase_dev.cxx", "THepMCParser_dev.h", "THepMCParser_dev.cxx"
    ]

    Packages = "\"VO_ALICE@Python-modules::1.0-27\",\n"
    if not LoadPackagesSeparately:
        Packages += "\"VO_ALICE@AliPhysics::{aliphysics}\",\n".format(
            aliphysics=AliPhysicsVersion)

    if "pythia8" in Gen:
        FilesToCopy.append("powheg_pythia8_conf.cmnd")

    if "powheg" in Gen:
        if OldPowhegInit:
            if PowhegStage == 0:
                alipowhegtools.main(yamlFileName, "./", Events, 0)
                FilesToCopy.extend([
                    "data/{}/pwggrid.dat".format(OldPowhegInit),
                    "data/{}/pwgubound.dat".format(OldPowhegInit)
                ])
            elif PowhegStage == 4:
                alipowhegtools.main(yamlFileName, "./", Events, 4)
                os.rename(alipowhegtools.GetParallelInputFileName(4),
                          "powheg.input")
                EssentialFilesToCopy = [
                    "pwggrid-????.dat", "pwggridinfo-btl-xg?-????.dat",
                    "pwgubound-????.dat"
                ]

                for fpattern in EssentialFilesToCopy:
                    for file in glob.glob("data/{}/{}".format(
                            OldPowhegInit, fpattern)):
                        FilesToCopy.append(file)

                seed_file_name = "pwgseeds.dat"
                FilesToDelete.append(seed_file_name)
                FilesToCopy.append(seed_file_name)
                with open(seed_file_name, "w") as seed_file:
                    for iseed in range(0, Jobs + 1):
                        seed_file.write(str(random.randint(0, 1073741824)))
                        seed_file.write("\n")
            else:
                logging.error(
                    "Not implemented for POWHEG stage {}".format(PowhegStage))
                exit(1)
        else:
            if PowhegStage != 0:
                logging.error(
                    "Not implemented for POWHEG stage {}".format(PowhegStage))
                exit(1)
            else:
                alipowhegtools.main(yamlFileName, "./", Events, 0)
        FilesToCopy.append("powheg.input")
        FilesToDelete.append("powheg.input")
        if not LoadPackagesSeparately:
            Packages += "\"VO_ALICE@POWHEG::r3178-alice1-1\",\n"
    if "herwig" in Gen:
        aliherwigtools.main(yamlFileName, "./", Events)
        FilesToCopy.extend([
            "herwig.in", "MB.in", "PPCollider.in", "SoftModel.in",
            "SoftTune.in"
        ])
        if HerwigTune:
            FilesToCopy.append(HerwigTune)
        FilesToDelete.append("herwig.in")
        if not LoadPackagesSeparately:
            Packages += "\"VO_ALICE@Herwig::v7.1.2-alice1-3\",\n"

    if PtHardList and len(PtHardList) > 1:
        minPtHardBin = 0
        maxPtHardBin = len(PtHardList) - 1
    else:
        minPtHardBin = -1
        maxPtHardBin = 0

    Packages = Packages[:-2]  # remove trailing ",\n"
    for ptHardBin in range(minPtHardBin, maxPtHardBin):
        if ptHardBin < 0:
            AlienDest = "{0}/{1}".format(AlienPath, TrainName)
            LocalDest = "{0}/{1}".format(LocalPath, TrainName)
            minPtHard = -1
            maxPtHard = -1
            JobsPtHard = Jobs
        else:
            minPtHard = PtHardList[ptHardBin]
            maxPtHard = PtHardList[ptHardBin + 1]
            AlienDest = "{0}/{1}/{2}".format(AlienPath, TrainName, ptHardBin)
            LocalDest = "{0}/{1}/{2}".format(LocalPath, TrainName, ptHardBin)
            JobsPtHard = Jobs[ptHardBin]
        JdlContent = GenerateProcessingJDL(ExeFile, AlienDest, Packages,
                                           ValidationScript, FilesToCopy, TTL,
                                           Events, JobsPtHard, yamlFileName,
                                           minPtHard, maxPtHard, PowhegStage)

        f = open(JdlFile, 'w')
        f.write(JdlContent)
        f.close()

        FilesToCopy.extend([JdlFile, ExeFile, ValidationScript])

        aligridtools.CopyFilesToTheGrid(FilesToCopy, AlienDest, LocalDest,
                                        Offline, GridUpdate)
        if not Offline:
            alisimtools.subprocess_call(
                ["alien_submit", "alien://{0}/{1}".format(AlienDest, JdlFile)])
        for file in FilesToDelete:
            os.remove(file)
    logging.info("Done.")

    alisimtools.subprocess_call(["ls", LocalDest])
Пример #5
0
def SubmitMergingJobs(TrainName, LocalPath, AlienPath, AliPhysicsVersion,
                      Offline, GridUpdate, TTL, MaxFilesPerJob, Gen, Proc,
                      PtHardList, MergingStage):
    if PtHardList and len(PtHardList) > 1:
        minPtHardBin = 0
        maxPtHardBin = len(PtHardList) - 1
    else:
        minPtHardBin = -1
        maxPtHardBin = 0

    for ptHardBin in range(minPtHardBin, maxPtHardBin):
        if ptHardBin < 0:
            TrainPtHardName = TrainName
        else:
            TrainPtHardName = "{0}/{1}".format(TrainName, ptHardBin)

        if MergingStage < 0:
            MergingStage = DetermineMergingStage(AlienPath, TrainPtHardName)

        if MergingStage < 0:
            logging.error(
                "Could not find any results from train {0}! Aborting...".
                format(TrainName))
            exit(1)
        elif MergingStage == 0:
            logging.info(
                "Merging stage determined to be 0 (i.e. first merging stage)")
            PreviousStagePath = "{0}/{1}/output".format(
                AlienPath, TrainPtHardName)
            SplitMethod = "parentdirectory"
        else:
            logging.info(
                "Merging stage determined to be {0}".format(MergingStage))
            PreviousStagePath = "{0}/{1}/stage_{2}/output".format(
                AlienPath, TrainPtHardName, MergingStage - 1)
            SplitMethod = "parentdirectory"

        AlienDest = "{0}/{1}/stage_{2}".format(AlienPath, TrainPtHardName,
                                               MergingStage)
        LocalDest = "{0}/{1}/stage_{2}".format(LocalPath, TrainPtHardName,
                                               MergingStage)

        if AlienFileExists(AlienDest): AlienDeleteDir(AlienDest)

        ValidationScript = "FastSim_validation.sh"
        ExeFile = "runFastSimMerging.py"
        JdlFile = "FastSim_Merging_{0}_{1}.jdl".format(Gen, Proc)
        XmlFile = "FastSim_Merging_{0}_{1}_stage_{2}.xml".format(
            Gen, Proc, MergingStage)

        FilesToCopy = ["runJetSimulationMergingGrid.C", "start_merging.C"]
        JdlContent = GenerateMergingJDL(ExeFile, XmlFile, AlienDest, TrainName,
                                        AliPhysicsVersion, ValidationScript,
                                        FilesToCopy, TTL, MaxFilesPerJob,
                                        SplitMethod)

        f = open(JdlFile, 'w')
        f.write(JdlContent)
        f.close()

        XmlContent = GenerateXMLCollection(PreviousStagePath, XmlFile)
        f = open(XmlFile, 'w')
        f.write(XmlContent)
        f.close()

        FilesToCopy.extend([JdlFile, XmlFile, ExeFile, ValidationScript])

        CopyFilesToTheGrid(FilesToCopy, AlienDest, LocalDest, Offline,
                           GridUpdate)
        if not Offline:
            alisimtools.subprocess_call(
                ["alien_submit", "alien://{0}/{1}".format(AlienDest, JdlFile)])
        os.remove(JdlFile)
        os.remove(XmlFile)
    logging.info("Done.")

    alisimtools.subprocess_call(["ls", LocalDest])
Пример #6
0
def AlienDeleteDir(fileName):
    if fileName.find("alien://") == -1:
        fname = fileName
    else:
        fname = fileName[8:]
    alisimtools.subprocess_call(["alien_rmdir", fname])