def add_local_site(sitecat, cp, local_path, local_url): """Add the local site to sitecatalog""" # local_url must end with a '/' if not local_url.endswith('/'): local_url = local_url + '/' local = Site("local", arch=Arch.X86_64, os_type=OS.LINUX) add_site_pegasus_profile(local, cp) local_dir = Directory(Directory.SHARED_SCRATCH, path=os.path.join(local_path, 'local-site-scratch')) local_file_serv = FileServer(urljoin(local_url, 'local-site-scratch'), Operation.ALL) local_dir.add_file_servers(local_file_serv) local.add_directories(local_dir) local.add_profiles(Namespace.PEGASUS, key="style", value="condor") local.add_profiles(Namespace.CONDOR, key="getenv", value="True") sitecat.add_sites(local)
def add_condorpool_symlink_site(sitecat, cp): """Add condorpool_symlink site to site catalog""" site = Site("condorpool_symlink", arch=Arch.X86_64, os_type=OS.LINUX) add_site_pegasus_profile(site, cp) site.add_profiles(Namespace.PEGASUS, key="style", value="condor") site.add_profiles(Namespace.PEGASUS, key="transfer.links", value="true") site.add_profiles(Namespace.PEGASUS, key="data.configuration", value="nonsharedfs") site.add_profiles(Namespace.PEGASUS, key='transfer.bypass.input.staging', value="true") site.add_profiles(Namespace.PEGASUS, key='auxillary.local', value="true") site.add_profiles(Namespace.CONDOR, key="+OpenScienceGrid", value="False") site.add_profiles(Namespace.CONDOR, key="should_transfer_files", value="Yes") site.add_profiles(Namespace.CONDOR, key="when_to_transfer_output", value="ON_EXIT_OR_EVICT") site.add_profiles(Namespace.CONDOR, key="getenv", value="True") site.add_profiles(Namespace.CONDOR, key="+DESIRED_Sites", value='"nogrid"') site.add_profiles(Namespace.CONDOR, key="+IS_GLIDEIN", value='"False"') site.add_profiles(Namespace.CONDOR, key="+flock_local", value="True") site.add_profiles(Namespace.DAGMAN, key="retry", value="2") sitecat.add_sites(site)
def add_condorpool_shared_site(sitecat, cp, local_path, local_url): """Add condorpool_shared site to site catalog""" # local_url must end with a '/' if not local_url.endswith('/'): local_url = local_url + '/' site = Site("condorpool_shared", arch=Arch.X86_64, os_type=OS.LINUX) add_site_pegasus_profile(site, cp) # It's annoying that this is needed! local_dir = Directory(Directory.SHARED_SCRATCH, path=os.path.join(local_path, 'cpool-site-scratch')) local_file_serv = FileServer(urljoin(local_url, 'cpool-site-scratch'), Operation.ALL) local_dir.add_file_servers(local_file_serv) site.add_directories(local_dir) site.add_profiles(Namespace.PEGASUS, key="style", value="condor") site.add_profiles(Namespace.PEGASUS, key="data.configuration", value="sharedfs") site.add_profiles(Namespace.PEGASUS, key='transfer.bypass.input.staging', value="true") site.add_profiles(Namespace.PEGASUS, key='auxillary.local', value="true") site.add_profiles(Namespace.CONDOR, key="+OpenScienceGrid", value="False") site.add_profiles(Namespace.CONDOR, key="should_transfer_files", value="Yes") site.add_profiles(Namespace.CONDOR, key="when_to_transfer_output", value="ON_EXIT_OR_EVICT") site.add_profiles(Namespace.CONDOR, key="getenv", value="True") site.add_profiles(Namespace.CONDOR, key="+DESIRED_Sites", value='"nogrid"') site.add_profiles(Namespace.CONDOR, key="+IS_GLIDEIN", value='"False"') site.add_profiles(Namespace.CONDOR, key="+flock_local", value="True") site.add_profiles(Namespace.DAGMAN, key="retry", value="2") # Need to set PEGASUS_HOME peg_home = distutils.spawn.find_executable('pegasus-plan') assert peg_home.endswith('bin/pegasus-plan') peg_home = peg_home.replace('bin/pegasus-plan', '') site.add_profiles(Namespace.ENV, key="PEGASUS_HOME", value=peg_home) sitecat.add_sites(site)
def add_osg_site(sitecat, cp): """Add osg site to site catalog""" site = Site("osg", arch=Arch.X86_64, os_type=OS.LINUX) add_site_pegasus_profile(site, cp) site.add_profiles(Namespace.PEGASUS, key="style", value="condor") site.add_profiles(Namespace.PEGASUS, key="data.configuration", value="nonsharedfs") site.add_profiles(Namespace.PEGASUS, key='transfer.bypass.input.staging', value="true") site.add_profiles(Namespace.CONDOR, key="should_transfer_files", value="Yes") site.add_profiles(Namespace.CONDOR, key="when_to_transfer_output", value="ON_EXIT_OR_EVICT") site.add_profiles(Namespace.CONDOR, key="+OpenScienceGrid", value="True") site.add_profiles(Namespace.CONDOR, key="getenv", value="False") site.add_profiles(Namespace.CONDOR, key="+InitializeModulesEnv", value="False") site.add_profiles(Namespace.CONDOR, key="+SingularityCleanEnv", value="True") site.add_profiles(Namespace.CONDOR, key="Requirements", value="(HAS_SINGULARITY =?= TRUE) && " "(HAS_LIGO_FRAMES =?= True) && " "(IS_GLIDEIN =?= True)") # FIXME: This one should be moved to be latest release and/or chosen in the # config file. site.add_profiles( Namespace.CONDOR, key="+SingularityImage", value='"/cvmfs/singularity.opensciencegrid.org/pycbc/pycbc-el7:v1.18.0"' ) # On OSG failure rate is high site.add_profiles(Namespace.DAGMAN, key="retry", value="4") site.add_profiles( Namespace.ENV, key="LAL_DATA_PATH", value= "/cvmfs/oasis.opensciencegrid.org/ligo/sw/pycbc/lalsuite-extra/current/share/lalsimulation" ) sitecat.add_sites(site)
def add_osg_site(sitecat, cp): """Add osg site to site catalog""" site = Site("osg", arch=Arch.X86_64, os_type=OS.LINUX) add_site_pegasus_profile(site, cp) site.add_profiles(Namespace.PEGASUS, key="style", value="condor") site.add_profiles(Namespace.PEGASUS, key="data.configuration", value="condorio") site.add_profiles(Namespace.PEGASUS, key='transfer.bypass.input.staging', value="true") site.add_profiles(Namespace.CONDOR, key="should_transfer_files", value="Yes") site.add_profiles(Namespace.CONDOR, key="when_to_transfer_output", value="ON_EXIT_OR_EVICT") site.add_profiles(Namespace.CONDOR, key="+OpenScienceGrid", value="True") site.add_profiles(Namespace.CONDOR, key="getenv", value="False") site.add_profiles(Namespace.CONDOR, key="preserve_relative_paths", value="True") site.add_profiles(Namespace.CONDOR, key="+InitializeModulesEnv", value="False") site.add_profiles(Namespace.CONDOR, key="+SingularityCleanEnv", value="True") site.add_profiles(Namespace.CONDOR, key="Requirements", value="(HAS_SINGULARITY =?= TRUE) && " "(HAS_LIGO_FRAMES =?= True) && " "(IS_GLIDEIN =?= True)") cvmfs_loc = '"/cvmfs/singularity.opensciencegrid.org/pycbc/pycbc-el8:v' cvmfs_loc += last_release + '"' site.add_profiles(Namespace.CONDOR, key="+SingularityImage", value=cvmfs_loc) # On OSG failure rate is high site.add_profiles(Namespace.DAGMAN, key="retry", value="4") site.add_profiles(Namespace.ENV, key="LAL_DATA_PATH", value="/cvmfs/oasis.opensciencegrid.org/ligo/sw/pycbc/lalsuite-extra/current/share/lalsimulation") # Add MKL location to LD_LIBRARY_PATH for OSG site.add_profiles(Namespace.ENV, key="LD_LIBRARY_PATH", value="/usr/local/lib:/.singularity.d/libs") sitecat.add_sites(site)