def add_saga_cluster_to_sites( sites_catalog: SiteCatalog, params: Parameters = Parameters.empty()) -> None: home = params.string("home_dir", default=str(Path.home().absolute())) data_configuration = params.string("data_configuration", default="sharedfs") shared_scratch_dir = params.string( "saga_shared_scratch", default=f"{home}/workflows/shared-scratch") saga = Site("saga", arch=Arch.X86_64, os_type=OS.LINUX) saga.add_directories( Directory(Directory.SHARED_SCRATCH, shared_scratch_dir).add_file_servers( FileServer("file://" + shared_scratch_dir, Operation.ALL))) saga.add_env(key="PEGASUS_HOME", value="/nas/gaia/shared/cluster/pegasus5/pegasus-5.0.0") # Profiles saga.add_pegasus_profile(style="glite", auxillary_local=True, data_configuration=data_configuration) saga.add_condor_profile(grid_resource="batch slurm") sites_catalog.add_sites(saga)
def add_condorpool_shared_site(sitecat, cp, local_path, local_url): """Add condorpool_shared site to site catalog""" # local_url must end with a '/' if not local_url.endswith('/'): local_url = local_url + '/' site = Site("condorpool_shared", arch=Arch.X86_64, os_type=OS.LINUX) add_site_pegasus_profile(site, cp) # It's annoying that this is needed! local_dir = Directory(Directory.SHARED_SCRATCH, path=os.path.join(local_path, 'cpool-site-scratch')) local_file_serv = FileServer(urljoin(local_url, 'cpool-site-scratch'), Operation.ALL) local_dir.add_file_servers(local_file_serv) site.add_directories(local_dir) site.add_profiles(Namespace.PEGASUS, key="style", value="condor") site.add_profiles(Namespace.PEGASUS, key="data.configuration", value="sharedfs") site.add_profiles(Namespace.PEGASUS, key='transfer.bypass.input.staging', value="true") site.add_profiles(Namespace.PEGASUS, key='auxillary.local', value="true") site.add_profiles(Namespace.CONDOR, key="+OpenScienceGrid", value="False") site.add_profiles(Namespace.CONDOR, key="should_transfer_files", value="Yes") site.add_profiles(Namespace.CONDOR, key="when_to_transfer_output", value="ON_EXIT_OR_EVICT") site.add_profiles(Namespace.CONDOR, key="getenv", value="True") site.add_profiles(Namespace.CONDOR, key="+DESIRED_Sites", value='"nogrid"') site.add_profiles(Namespace.CONDOR, key="+IS_GLIDEIN", value='"False"') site.add_profiles(Namespace.CONDOR, key="+flock_local", value="True") site.add_profiles(Namespace.DAGMAN, key="retry", value="2") # Need to set PEGASUS_HOME peg_home = which('pegasus-plan') assert peg_home.endswith('bin/pegasus-plan') peg_home = peg_home.replace('bin/pegasus-plan', '') site.add_profiles(Namespace.ENV, key="PEGASUS_HOME", value=peg_home) sitecat.add_sites(site)
def add_local_site(sitecat, cp, local_path, local_url): """Add the local site to sitecatalog""" # local_url must end with a '/' if not local_url.endswith('/'): local_url = local_url + '/' local = Site("local", arch=Arch.X86_64, os_type=OS.LINUX) add_site_pegasus_profile(local, cp) local_dir = Directory(Directory.SHARED_SCRATCH, path=os.path.join(local_path, 'local-site-scratch')) local_file_serv = FileServer(urljoin(local_url, 'local-site-scratch'), Operation.ALL) local_dir.add_file_servers(local_file_serv) local.add_directories(local_dir) local.add_profiles(Namespace.PEGASUS, key="style", value="condor") local.add_profiles(Namespace.CONDOR, key="getenv", value="True") sitecat.add_sites(local)
def add_local_nas_to_sites( sites_catalog: SiteCatalog, params: Parameters = Parameters.empty()) -> None: home = params.string("home_dir", default=str(Path.home().absolute())) shared_scratch_dir = params.string("local_shared_scratch", default=f"{home}/workflows/scratch") local_storage_dir = params.string("local_storage", default=f"{home}/workflows/output") sites_catalog.add_sites( Site("local", arch=Arch.X86_64, os_type=OS.LINUX).add_directories( Directory(Directory.SHARED_SCRATCH, shared_scratch_dir).add_file_servers( FileServer("file://" + shared_scratch_dir, Operation.ALL)), Directory(Directory.LOCAL_STORAGE, local_storage_dir).add_file_servers( FileServer("file://" + local_storage_dir, Operation.ALL)), ))