示例#1
0
async def bake(
    root: Root,
    batch: str,
    local_executor: bool,
    meta_from_file: Optional[str],
    param: List[Tuple[str, str]],
    name: Optional[str],
    tag: Sequence[str],
) -> None:
    """Start a batch.

    Run BATCH pipeline remotely on the cluster.
    """
    async with AsyncExitStack() as stack:
        client = await stack.enter_async_context(neuro_sdk.get())
        storage: Storage = await stack.enter_async_context(ApiStorage(client))
        runner = await stack.enter_async_context(
            BatchRunner(root.config_dir, root.console, client, storage, root))
        params = {key: value for key, value in param}
        if meta_from_file is not None:
            bake_meta = parse_bake_meta(LocalPath(meta_from_file))
            params = {**bake_meta, **params}
        await runner.bake(
            batch_name=batch,
            local_executor=local_executor,
            params=params,
            name=name,
            tags=tag,
        )
示例#2
0
 async def async_shell_complete(self, root: Root, ctx: click.Context,
                                param: click.Parameter,
                                incomplete: str) -> List[CompletionItem]:
     variants: List[str] = []
     bake_id = ctx.params[self._bake_id_param_name]
     attempt_no = ctx.params[self._attempt_no_param_name]
     async with AsyncExitStack() as stack:
         client = await stack.enter_async_context(neuro_sdk.get())
         storage: Storage = await stack.enter_async_context(
             ApiStorage(client))
         runner: BatchRunner = await stack.enter_async_context(
             BatchRunner(root.config_dir, root.console, client, storage,
                         root))
         try:
             bake_id = await resolve_bake(bake_id,
                                          project=runner.project_id,
                                          storage=storage)
             attempt = await runner.get_bake_attempt(bake_id,
                                                     attempt_no=attempt_no)
         except ResourceNotFound:
             return []
         tasks = [
             task async for task in storage.bake(id=bake_id).attempt(
                 id=attempt.id).list_tasks()
         ]
         if self._include_finished:
             variants.extend(".".join(task.yaml_id) for task in tasks
                             if task.status.is_finished)
         if self._include_started:
             variants.extend(".".join(task.yaml_id) for task in tasks
                             if task.status.is_finished)
     return [
         CompletionItem(task) for task in variants
         if task.startswith(incomplete)
     ]
示例#3
0
async def restart(
    root: Root,
    bake: str,
    attempt: int,
    from_failed: bool,
    local_executor: bool,
) -> None:
    """Start a batch.

    Run BATCH pipeline remotely on the cluster.
    """
    async with AsyncExitStack() as stack:
        client = await stack.enter_async_context(neuro_sdk.get())
        storage: Storage = await stack.enter_async_context(ApiStorage(client))
        runner = await stack.enter_async_context(
            BatchRunner(root.config_dir, root.console, client, storage, root))
        bake_id = await resolve_bake(bake,
                                     project=runner.project_id,
                                     storage=storage)
        await runner.restart(
            bake_id,
            attempt_no=attempt,
            from_failed=from_failed,
            local_executor=local_executor,
        )
示例#4
0
async def inspect(
    root: Root,
    bake: str,
    attempt: int,
    output_graph: Optional[str],
    dot: bool,
    pdf: bool,
    view: bool,
) -> None:
    """Inspect a bake.

    Display a list of started/finished tasks of BAKE\\_ID.
    """
    async with AsyncExitStack() as stack:
        client = await stack.enter_async_context(neuro_sdk.get())
        storage: Storage = await stack.enter_async_context(ApiStorage(client))
        runner = await stack.enter_async_context(
            BatchRunner(root.config_dir, root.console, client, storage, root))
        if output_graph is not None:
            real_output: Optional[LocalPath] = LocalPath(output_graph)
        else:
            real_output = None
        bake_id = await resolve_bake(bake,
                                     project=runner.project_id,
                                     storage=storage)

        await runner.inspect(
            bake_id,
            attempt_no=attempt,
            output=real_output,
            save_dot=dot,
            save_pdf=pdf,
            view_pdf=view,
        )
示例#5
0
async def execute(
    root: Root,
    bake_id: str,
) -> None:
    """Start a batch.

    Run BATCH pipeline remotely on the cluster.
    """
    # neuro-flow execute is run in linux container only,
    # Linux signals are always defined.
    for signame in (
            signal.SIGHUP,
            signal.SIGINT,
            signal.SIGQUIT,
            signal.SIGTSTP,
            signal.SIGTERM,
            signal.SIGTTIN,
            signal.SIGTTOU,
            signal.SIGWINCH,
    ):
        # ignore everything, use neuro-flow cancel to stop the master job.
        signal.signal(signame, signal.SIG_IGN)
    async with AsyncExitStack() as stack:
        client = await stack.enter_async_context(neuro_sdk.get())
        storage: Storage = await stack.enter_async_context(ApiStorage(client))
        runner = await stack.enter_async_context(
            BatchRunner(root.config_dir, root.console, client, storage, root))
        await runner.process(bake_id)
示例#6
0
async def mkvolumes(root: Root, ) -> None:
    """Create all remote folders for volumes."""
    async with AsyncExitStack() as stack:
        client = await stack.enter_async_context(neuro_sdk.get())
        storage: Storage = await stack.enter_async_context(ApiStorage(client))
        runner = await stack.enter_async_context(
            LiveRunner(root.config_dir, root.console, client, storage, root))
        await runner.mkvolumes()
示例#7
0
async def build(root: Root, force_overwrite: bool, image: str) -> None:
    """Build an image.

    Assemble the IMAGE remotely and publish it.
    """
    async with AsyncExitStack() as stack:
        client = await stack.enter_async_context(neuro_sdk.get())
        storage: Storage = await stack.enter_async_context(ApiStorage(client))
        runner = await stack.enter_async_context(
            LiveRunner(root.config_dir, root.console, client, storage, root))
        if image == "ALL":
            await runner.build_all(force_overwrite=force_overwrite)
        else:
            await runner.build(image, force_overwrite=force_overwrite)
示例#8
0
 async def async_shell_complete(self, root: Root, ctx: click.Context,
                                param: click.Parameter,
                                incomplete: str) -> List[CompletionItem]:
     job_id = ctx.params[self._job_id_param_name]
     async with AsyncExitStack() as stack:
         client = await stack.enter_async_context(neuro_sdk.get())
         storage: Storage = await stack.enter_async_context(
             ApiStorage(client))
         runner = await stack.enter_async_context(
             LiveRunner(root.config_dir, root.console, client, storage,
                        root))
         variants = await runner.list_suffixes(job_id)
     return [
         CompletionItem(suffix) for suffix in variants
         if suffix.startswith(incomplete)
     ]
示例#9
0
 async def async_shell_complete(self, root: Root, ctx: click.Context,
                                param: click.Parameter,
                                incomplete: str) -> List[CompletionItem]:
     async with AsyncExitStack() as stack:
         client = await stack.enter_async_context(neuro_sdk.get())
         storage: Storage = await stack.enter_async_context(
             ApiStorage(client))
         runner = await stack.enter_async_context(
             LiveRunner(root.config_dir, root.console, client, storage,
                        root))
         variants = list(runner.flow.job_ids)
         if self._allow_all:
             variants += ["ALL"]
     return [
         CompletionItem(job_id) for job_id in variants
         if job_id.startswith(incomplete)
     ]
示例#10
0
 async def async_shell_complete(self, root: Root, ctx: click.Context,
                                param: click.Parameter,
                                incomplete: str) -> List[CompletionItem]:
     variants = []
     async with AsyncExitStack() as stack:
         client = await stack.enter_async_context(neuro_sdk.get())
         storage: Storage = await stack.enter_async_context(
             ApiStorage(client))
         try:
             async for project in storage.list_projects():
                 variants.append(project.yaml_id)
         except ValueError:
             pass
     return [
         CompletionItem(yaml_id) for yaml_id in variants
         if yaml_id.startswith(incomplete)
     ]
示例#11
0
async def download(
    root: Root,
    volume: str,
) -> None:
    """Download volume.

    Download remote files to local for VOLUME,
    use `download ALL` for downloading all volumes."""
    async with AsyncExitStack() as stack:
        client = await stack.enter_async_context(neuro_sdk.get())
        storage: Storage = await stack.enter_async_context(ApiStorage(client))
        runner = await stack.enter_async_context(
            LiveRunner(root.config_dir, root.console, client, storage, root))
        if volume != "ALL":
            await runner.download(volume)
        else:
            await runner.download_all()
示例#12
0
async def clean(
    root: Root,
    volume: str,
) -> None:
    """Clean volume.

    Clean remote files on VOLUME,
    use `clean ALL` for cleaning up all volumes."""
    async with AsyncExitStack() as stack:
        client = await stack.enter_async_context(neuro_sdk.get())
        storage: Storage = await stack.enter_async_context(ApiStorage(client))
        runner = await stack.enter_async_context(
            LiveRunner(root.config_dir, root.console, client, storage, root))
        if volume != "ALL":
            await runner.clean(volume)
        else:
            await runner.clean_all()
示例#13
0
async def cancel(
    root: Root,
    bake: str,
    attempt: int,
) -> None:
    """Cancel a bake.

    Cancel a bake execution by stopping all started tasks.
    """
    async with AsyncExitStack() as stack:
        client = await stack.enter_async_context(neuro_sdk.get())
        storage: Storage = await stack.enter_async_context(ApiStorage(client))
        runner = await stack.enter_async_context(
            BatchRunner(root.config_dir, root.console, client, storage, root))
        bake_id = await resolve_bake(bake,
                                     project=runner.project_id,
                                     storage=storage)
        await runner.cancel(bake_id, attempt_no=attempt)
示例#14
0
async def bakes(
    root: Root,
    tag: Sequence[str],
    since: Optional[str],
    until: Optional[str],
    recent_first: bool,
) -> None:
    """List existing bakes."""
    async with AsyncExitStack() as stack:
        client = await stack.enter_async_context(neuro_sdk.get())
        storage: Storage = await stack.enter_async_context(ApiStorage(client))
        runner = await stack.enter_async_context(
            BatchRunner(root.config_dir, root.console, client, storage, root))
        await runner.list_bakes(
            tags=set(tag),
            since=_parse_date(since),
            until=_parse_date(until),
            recent_first=recent_first,
        )
示例#15
0
async def show(
    root: Root,
    bake: str,
    attempt: int,
    task_id: str,
    raw: bool,
) -> None:
    """Show output of baked task.

    Display a logged output of TASK\\_ID from BAKE\\_ID.
    """
    async with AsyncExitStack() as stack:
        client = await stack.enter_async_context(neuro_sdk.get())
        storage: Storage = await stack.enter_async_context(ApiStorage(client))
        runner = await stack.enter_async_context(
            BatchRunner(root.config_dir, root.console, client, storage, root))
        bake_id = await resolve_bake(bake,
                                     project=runner.project_id,
                                     storage=storage)
        await runner.logs(bake_id, task_id, attempt_no=attempt, raw=raw)
示例#16
0
 async def async_shell_complete(self, root: Root, ctx: click.Context,
                                param: click.Parameter,
                                incomplete: str) -> List[CompletionItem]:
     async with AsyncExitStack() as stack:
         client = await stack.enter_async_context(neuro_sdk.get())
         storage: Storage = await stack.enter_async_context(
             ApiStorage(client))
         runner = await stack.enter_async_context(
             LiveRunner(root.config_dir, root.console, client, storage,
                        root))
         variants = [
             volume.id for volume in runner.flow.volumes.values()
             if volume.local is not None
         ]
         if self._allow_all:
             variants += ["ALL"]
     return [
         CompletionItem(image) for image in variants
         if image.startswith(incomplete)
     ]
示例#17
0
 async def async_shell_complete(self, root: Root, ctx: click.Context,
                                param: click.Parameter,
                                incomplete: str) -> List[CompletionItem]:
     variants = []
     async with AsyncExitStack() as stack:
         client = await stack.enter_async_context(neuro_sdk.get())
         storage: Storage = await stack.enter_async_context(
             ApiStorage(client))
         runner: BatchRunner = await stack.enter_async_context(
             BatchRunner(root.config_dir, root.console, client, storage,
                         root))
         try:
             async for bake in runner.get_bakes():
                 variants.append(bake.id)
                 if bake.name is not None:
                     variants.append(bake.name)
         except ValueError:
             pass
     return [
         CompletionItem(bake) for bake in variants
         if bake.startswith(incomplete)
     ]
示例#18
0
async def clear_cache(
    root: Root,
    batch: str,
    task_id: Optional[str],
) -> None:
    """Clear cache.

    Use `neuro-flow clear-cache <BATCH>` for cleaning up the cache for BATCH;
    Use `neuro-flow clear-cache <BATCH> <TASK_ID>` for cleaning up the cache
    for TASK_ID in BATCH;

    `neuro-flow clear-cache ALL` clears all caches.
    """
    async with AsyncExitStack() as stack:
        client = await stack.enter_async_context(neuro_sdk.get())
        storage: Storage = await stack.enter_async_context(ApiStorage(client))
        runner = await stack.enter_async_context(
            BatchRunner(root.config_dir, root.console, client, storage, root))
        if batch == "ALL":
            await runner.clear_cache(None)
        else:
            await runner.clear_cache(batch, task_id)