def add_saga_cluster_to_sites(
sites_catalog: SiteCatalog,
params: Parameters = Parameters.empty()) -> None:
home = params.string("home_dir", default=str(Path.home().absolute()))
data_configuration = params.string("data_configuration",
default="sharedfs")
shared_scratch_dir = params.string(
"saga_shared_scratch", default=f"{home}/workflows/shared-scratch")
saga = Site("saga", arch=Arch.X86_64, os_type=OS.LINUX)
saga.add_directories(
Directory(Directory.SHARED_SCRATCH,
shared_scratch_dir).add_file_servers(
FileServer("file://" + shared_scratch_dir,
Operation.ALL)))
saga.add_env(key="PEGASUS_HOME",
value="/nas/gaia/shared/cluster/pegasus5/pegasus-5.0.0")
# Profiles
saga.add_pegasus_profile(style="glite",
auxillary_local=True,
data_configuration=data_configuration)
saga.add_condor_profile(grid_resource="batch slurm")
sites_catalog.add_sites(saga)
def add_condorpool_shared_site(sitecat, cp, local_path, local_url):
"""Add condorpool_shared site to site catalog"""
# local_url must end with a '/'
if not local_url.endswith('/'):
local_url = local_url + '/'
site = Site("condorpool_shared", arch=Arch.X86_64, os_type=OS.LINUX)
add_site_pegasus_profile(site, cp)
# It's annoying that this is needed!
local_dir = Directory(Directory.SHARED_SCRATCH,
path=os.path.join(local_path, 'cpool-site-scratch'))
local_file_serv = FileServer(urljoin(local_url, 'cpool-site-scratch'),
Operation.ALL)
local_dir.add_file_servers(local_file_serv)
site.add_directories(local_dir)
site.add_profiles(Namespace.PEGASUS, key="style", value="condor")
site.add_profiles(Namespace.PEGASUS, key="data.configuration",
value="sharedfs")
site.add_profiles(Namespace.PEGASUS, key='transfer.bypass.input.staging',
value="true")
site.add_profiles(Namespace.PEGASUS, key='auxillary.local',
value="true")
site.add_profiles(Namespace.CONDOR, key="+OpenScienceGrid",
value="False")
site.add_profiles(Namespace.CONDOR, key="should_transfer_files",
value="Yes")
site.add_profiles(Namespace.CONDOR, key="when_to_transfer_output",
value="ON_EXIT_OR_EVICT")
site.add_profiles(Namespace.CONDOR, key="getenv", value="True")
site.add_profiles(Namespace.CONDOR, key="+DESIRED_Sites",
value='"nogrid"')
site.add_profiles(Namespace.CONDOR, key="+IS_GLIDEIN",
value='"False"')
site.add_profiles(Namespace.CONDOR, key="+flock_local",
value="True")
site.add_profiles(Namespace.DAGMAN, key="retry", value="2")
# Need to set PEGASUS_HOME
peg_home = which('pegasus-plan')
assert peg_home.endswith('bin/pegasus-plan')
peg_home = peg_home.replace('bin/pegasus-plan', '')
site.add_profiles(Namespace.ENV, key="PEGASUS_HOME", value=peg_home)
sitecat.add_sites(site)
def add_local_site(sitecat, cp, local_path, local_url):
"""Add the local site to sitecatalog"""
# local_url must end with a '/'
if not local_url.endswith('/'):
local_url = local_url + '/'
local = Site("local", arch=Arch.X86_64, os_type=OS.LINUX)
add_site_pegasus_profile(local, cp)
local_dir = Directory(Directory.SHARED_SCRATCH,
path=os.path.join(local_path, 'local-site-scratch'))
local_file_serv = FileServer(urljoin(local_url, 'local-site-scratch'),
Operation.ALL)
local_dir.add_file_servers(local_file_serv)
local.add_directories(local_dir)
local.add_profiles(Namespace.PEGASUS, key="style", value="condor")
local.add_profiles(Namespace.CONDOR, key="getenv", value="True")
sitecat.add_sites(local)
