Python String Exemples

Exemple #1

import pandas as pd

from suanpan import asyncio, path, utils
from suanpan.arguments import String
from suanpan.docker import DockerComponent as dc
from suanpan.docker.arguments import Folder, HiveTable


@dc.input(
    HiveTable(key="inputData",
              table="inputDataTable",
              partition="inputDataPartition"))
@dc.input(Folder(key="inputDataFolder", required=True))
@dc.output(Folder(key="outputImagesFolder", required=True))
@dc.column(String(key="idColumn", default="id"))
@dc.column(String(key="dataColumn", default="data_path"))
def SPData2Images(context):
    args = context.args

    with asyncio.multiThread() as pool:
        for _, row in args.inputData.iterrows():
            image = utils.loadFromNpy(
                os.path.join(args.inputDataFolder, row[args.dataColumn]))
            prefix = os.path.join(args.outputImagesFolder, row[args.idColumn])
            utils.saveAllAsImages(prefix, image, pool=pool)

    return args.outputImagesFolder


if __name__ == "__main__":

Exemple #2

    Folder(
        key="inputDataFolder",
        required=True,
        help="DSB stage1/2 or similar directory path.",
    ))
@dc.output(
    HiveTable(key="outputData",
              table="outputDataTable",
              partition="outputDataPartition"))
@dc.output(
    Folder(
        key="outputDataFolder",
        required=True,
        help="Directory to save preprocessed npy files to.",
    ))
@dc.output(String(key="idColumn", default="patient"))
@dc.output(String(key="imageColumn", default="image_path"))
def SPPredictPreprocess(context):
    args = context.args

    stage1Path = args.inputDataFolder
    preprocessResultPath = args.outputDataFolder

    full_prep(stage1Path,
              preprocessResultPath,
              n_worker=asyncio.WORKERS,
              use_existing=False)
    data = scan_prep_results(preprocessResultPath, args.idColumn,
                             args.imageColumn)

    return data, args.outputDataFolder

Exemple #3

@dc.input(
    Checkpoint(key="inputCheckpoint", required=True, help="Ckpt model file."))
@dc.output(
    HiveTable(
        key="outputBboxData",
        table="outputBboxDataTable",
        partition="outputBboxDataPartition",
    ))
@dc.output(
    Folder(
        key="outputBboxFolder",
        required=True,
        help="Directory to save bbox npy files to.",
    ))
@dc.column(
    String(key="idColumn", default="id", help="ID column of inputImagesData."))
@dc.column(
    String(key="lbbColumn",
           default="lbb_path",
           help="Lbb column of inputImagesData."))
@dc.column(
    String(key="pbbColumn",
           default="pbb_path",
           help="Pbb column of inputImagesData."))
@dc.param(Int(key="margin", default=16, help="patch margin."))
@dc.param(Int(key="sidelen", default=64, help="patch side length."))
@dc.param(Int(key="batchSize", default=16))
def SPNNetPredict(context):
    args = context.args

    data = args.inputData

Exemple #4

@dc.input(
    HiveTable(
        key="inputTrainData",
        table="inputTrainDataTable",
        partition="inputTrainDataPartition",
    ))
@dc.input(
    HiveTable(
        key="inputValidateData",
        table="inputValidateDataTable",
        partition="inputValidateDataPartition",
    ))
@dc.input(Folder(key="inputDataFolder", required=True))
@dc.input(Checkpoint(key="inputCheckpoint"))
@dc.output(Checkpoint(key="outputCheckpoint", required=True))
@dc.column(String(key="idColumn", default="id"))
@dc.param(Int(key="epochs", default=100))
@dc.param(Int(key="batchSize", default=16))
@dc.param(Float(key="learningRate", default=0.01))
@dc.param(Float(key="momentum", default=0.9))
@dc.param(Float(key="weightDecay", default=1e-4))
@dc.param(Bool(key="distributed", default=False))
@dc.param(String(key="ckptFolder"))
@dc.param(Bool(key="saveFreq", default=1))
def SPNNetTrain(context):
    torch.manual_seed(0)

    args = context.args

    saveFolder = os.path.dirname(args.outputCheckpoint)
    dataFolder = args.inputDataFolder

Exemple #5

@dc.input(
    HiveTable(
        key="inputTrainData",
        table="inputTrainDataTable",
        partition="inputTrainDataPartition",
    ))
@dc.input(
    HiveTable(
        key="inputValidateData",
        table="inputValidateDataTable",
        partition="inputValidateDataPartition",
    ))
@dc.input(Folder(key="inputDataFolder", required=True))
@dc.input(Checkpoint(key="inputCheckpoint"))
@dc.output(Checkpoint(key="outputCheckpoint", required=True))
@dc.column(String(key="idColumn", default="id"))
@dc.param(Int(key="epochs", default=100))
@dc.param(Int(key="batchSize", default=16))
@dc.param(Float(key="learningRate", default=0.01))
@dc.param(Float(key="momentum", default=0.9))
@dc.param(Float(key="weightDecay", default=1e-4))
def SPNNetTrain(context):
    torch.manual_seed(0)

    args = context.args

    saveFolder = os.path.dirname(args.outputCheckpoint)
    dataFolder = args.inputDataFolder
    checkoutPointPath = args.inputCheckpoint
    trainIds = args.inputTrainData[args.idColumn]
    validateIds = args.inputValidateData[args.idColumn]