def __init__(
self,
grid_type: _GridType,
contact: str,
scheduler_type: Scheduler,
job_type: Optional[SupportedJobs] = None,
):
"""
:param grid_type: a grid type defined in :py:class:`~Pegasus.api.site_catalog.Grid` (e.g. :code:`Grid.SLURM`)
:type grid_type: _GridType
:param contact: endpoint such as :code:`"workflow.isi.edu"`
:type contact: str
:param scheduler_type: a scheduler type defined in :py:class:`~Pegasus.api.site_catalog.Scheduler` (e.g. :code:`Scheduler.SLURM`)
:type scheduler_type: Scheduler
:param job_type: a job type defined in :py:class:`~Pegasus.api.site_catalog.SupportedJobs` (e.g. :code:`SupportedJobs.COMPUTE`), defaults to None
:type job_type: Optional[SupportedJobs], optional
:raises ValueError: grid_type must be one defined in :py:class:`~Pegasus.api.site_catalog.Grid` (e.g. :code:`Grid.PBS`)
:raises ValueError: scheduler_type must be one defined in :py:class:`~Pegasus.api.site_catalog.Scheduler` (e.g. :code:`Scheduler.PBS`)
:raises ValueError: job_type must be one defined in :py:class`~Pegasus.api.site_catalog.SupportedJobs` (e.g. :code:`SupportedJobs.COMPUTE`)
"""
if not isinstance(grid_type, _GridType):
raise TypeError(
"invalid grid_type: {grid_type}; grid_type must be one of {cls_enum_str}"
.format(
grid_type=grid_type,
cls_enum_str=_get_class_enum_member_str(Grid, _GridType),
))
self.grid_type = grid_type.value
self.contact = contact
if not isinstance(scheduler_type, Scheduler):
raise TypeError(
"invalid scheduler_type: {scheduler_type}; scheduler_type must be one of {enum_str}"
.format(scheduler_type=scheduler_type,
enum_str=_get_enum_str(Scheduler)))
self.scheduler_type = scheduler_type.value
if job_type is not None:
if not isinstance(job_type, SupportedJobs):
raise TypeError(
"invalid job_type: {job_type}; job_type must be one of {enum_str}"
.format(job_type=job_type,
enum_str=_get_enum_str(SupportedJobs)))
else:
self.job_type = job_type.value
else:
self.job_type = None
def __init__(
self,
name: str,
arch: Optional[Arch] = None,
os_type: Optional[OS] = None,
os_release: Optional[str] = None,
os_version: Optional[str] = None,
):
"""
:param name: name of the site
:type name: str
:param arch: the site's architecture (e.g. :code:`Arch.X86_64`), defaults to None
:type arch: Optional[Arch]
:param os_type: the site's operating system (e.g. :code:`OS.LINUX`), defaults to None
:type os_type: Optional[OS], optional
:param os_release: the release of the site's operating system, defaults to None
:type os_release: Optional[str]
:param os_version: the version of the site's operating system, defaults to None
:type os_version: Optional[str]
:raises ValueError: arch must be one of :py:class:`~Pegasus.api.site_catalog.Arch`
:raises ValueError: os_type must be one of :py:class:`~Pegasus.api.site_catalog.OS`
"""
self.name = name
self.directories = list()
self.grids = list()
if arch is not None:
if not isinstance(arch, Arch):
raise TypeError(
"invalid arch: {arch}; arch must be one of {enum_str}".
format(arch=arch, enum_str=_get_enum_str(Arch)))
else:
self.arch = arch.value
else:
self.arch = arch
if os_type is not None:
if not isinstance(os_type, OS):
raise TypeError(
"invalid os_type: {os_type}; os_type must be one of {enum_str}"
.format(os_type=os_type, enum_str=_get_enum_str(OS)))
else:
self.os_type = os_type.value
else:
self.os_type = os_type
self.os_release = os_release
self.os_version = os_version
self.profiles = defaultdict(dict)
def test__get_enum_str():
class Constants(Enum):
A = "a"
B = "b"
result = _get_enum_str(Constants)
expected = "Constants.<A | B>"
assert result == expected
def __init__(self, url: str, operation_type: Operation):
"""
:param url: url including protocol such as :code:`scp://obelix.isi.edu/data`
:type url: str
:param operation_type: operation type defined in :py:class:`~Pegasus.api.site_catalog.OperationType` (e.g. :code:`Operation.ALL`)
:type operation_type: OperationType
:raises ValueError: operation_type must be one defined in :py:class:`~Pegasus.api.site_catalog.OperationType`
"""
self.url = url
if not isinstance(operation_type, Operation):
raise TypeError(
"invalid operation_type: {operation_type}; operation_type must be one of {enum_str}"
.format(operation_type=operation_type,
enum_str=_get_enum_str(Operation)))
self.operation_type = operation_type.value
self.profiles = defaultdict(dict)
def __init__(
self,
name: str,
pfn: Union[str, Path],
is_stageable: bool = False,
bypass_staging: bool = False,
arch: Optional[Arch] = None,
os_type: Optional[OS] = None,
os_release: Optional[str] = None,
os_version: Optional[str] = None,
container: Optional[Union[Container, str]] = None,
):
"""
:param name: name of the site at which this :py:class:`~Pegasus.api.transformation_catalog.Transformation` resides
:type name: str
:param pfn: physical file name, an absolute path given as a str or Path
:type pfn: Union[str, Path]
:param is_stageable: whether or not this transformation is stageable or installed, defaults to False
:type type: bool, optional
:param bypass_staging: whether or not to bypass the stage in job of this executable (Note that this only works for transformations where :code:`is_stageable=False`), defaults to False
:type bypass_staging: bool, optional
:param arch: architecture that this :py:class:`~Pegasus.api.transformation_catalog.Transformation` was compiled for (defined in :py:class:`~Pegasus.api.site_catalog.Arch`), defaults to None
:type arch: Optional[Arch], optional
:param os_type: name of os that this :py:class:`~Pegasus.api.transformation_catalog.Transformation` was compiled for (defined in :py:class:`~Pegasus.api.site_catalog.OS`), defaults to None
:type os_type: Optional[OS], optional
:param os_release: release of os that this :py:class:`~Pegasus.api.transformation_catalog.Transformation` was compiled for, defaults to None, defaults to None
:type os_release: Optional[str], optional
:param os_version: version of os that this :py:class:`~Pegasus.api.transformation_catalog.Transformation` was compiled for, defaults to None, defaults to None
:type os_version: Optional[str], optional
:param container: specify the name of the container or Container object to use, optional
:type container: Optional[Union[Container, str]], optional
:raises TypeError: arch must be one of :py:class:`~Pegasus.api.site_catalog.Arch`
:raises TypeError: os_type must be one of :py:class:`~Pegasus.api.site_catalog.OS`
:raises ValueError: pfn must be given as an absolute path when :code:`pathlib.Path` is used
:raises ValueError: :code:`bypass_staging=True` can only be used when :code:`is_stageable=True`
"""
self.name = name
if isinstance(pfn, Path):
if not pfn.is_absolute():
raise ValueError(
"invalid pfn: {}, the given pfn must be an absolute path".
format(str(pfn)))
pfn = str(pfn)
self.pfn = pfn
self.transformation_type = "stageable" if is_stageable else "installed"
self.bypass = None
if bypass_staging:
if not is_stageable:
raise ValueError(
"bypass_staging can only be used when is_stageable is set to True"
)
self.bypass = bypass_staging
if arch is not None:
if not isinstance(arch, Arch):
raise TypeError(
"invalid arch: {arch}; arch must be one of {enum_str}".
format(arch=arch, enum_str=_get_enum_str(Arch)))
else:
self.arch = arch.value
else:
self.arch = None
if os_type is not None:
if not isinstance(os_type, OS):
raise TypeError(
"invalid os_type: {os_type}; os_type must be one of {enum_str}"
.format(os_type=os_type, enum_str=_get_enum_str(OS)))
else:
self.os_type = os_type.value
else:
self.os_type = None
self.os_release = os_release
self.os_version = os_version
container_name = None
if container:
if isinstance(container, Container):
container_name = container.name
elif isinstance(container, str):
container_name = container
else:
raise TypeError(
"invalid container: {container}; container must be of type Container or str (the container name)"
.format(container=container))
self.container = container_name
self.profiles = defaultdict(dict)
self.metadata = dict()
def test_invalid__get_enum_str():
with pytest.raises(TypeError) as e:
_get_enum_str(int)
assert "invalid enum_cls: {_type}".format(_type=int) in str(e)
def __init__(
self,
grid_type,
contact,
scheduler_type,
job_type=None,
free_mem=None,
total_mem=None,
max_count=None,
max_cpu_time=None,
running_jobs=None,
jobs_in_queue=None,
idle_nodes=None,
total_nodes=None,
):
# TODO: get descriptions for params
"""
:param grid_type: [description]
:type grid_type: [type]
:param contact: [description]
:type contact: [type]
:param scheduler_type: [description]
:type scheduler_type: [type]
:param job_type: [description], defaults to None
:type job_type: [type], optional
:param free_mem: [description], defaults to None
:type free_mem: [type], optional
:param total_mem: [description], defaults to None
:type total_mem: [type], optional
:param max_count: [description], defaults to None
:type max_count: [type], optional
:param max_cpu_time: [description], defaults to None
:type max_cpu_time: [type], optional
:param running_jobs: [description], defaults to None
:type running_jobs: [type], optional
:param jobs_in_queue: [description], defaults to None
:type jobs_in_queue: [type], optional
:param idle_nodes: [description], defaults to None
:type idle_nodes: [type], optional
:param total_nodes: [description], defaults to None
:type total_nodes: [type], optional
:raises ValueError:
:raises ValueError:
:raises ValueError:
"""
if not isinstance(grid_type, _GridType):
raise TypeError(
"invalid grid_type: {grid_type}; grid_type must be one of {cls_enum_str}".format(
grid_type=grid_type,
cls_enum_str=_get_class_enum_member_str(Grid, _GridType),
)
)
self.grid_type = grid_type.value
self.contact = contact
if not isinstance(scheduler_type, Scheduler):
raise TypeError(
"invalid scheduler_type: {scheduler_type}; scheduler_type must be one of {enum_str}".format(
scheduler_type=scheduler_type, enum_str=_get_enum_str(Scheduler)
)
)
self.scheduler_type = scheduler_type.value
if job_type is not None:
if not isinstance(job_type, SupportedJobs):
raise TypeError(
"invalid job_type: {job_type}; job_type must be one of {enum_str}".format(
job_type=job_type, enum_str=_get_enum_str(SupportedJobs)
)
)
else:
self.job_type = job_type.value
else:
self.job_type = None
self.free_mem = free_mem
self.total_mem = total_mem
self.max_count = max_count
self.max_cpu_time = max_cpu_time
self.running_jobs = running_jobs
self.jobs_in_queue = jobs_in_queue
self.idle_nodes = idle_nodes
self.total_nodes = total_nodes