Ejemplo n.º 1
0
gpu_compute_target.wait_for_completion(show_output=True)

# Get datastore reference
datastore = DataReference(datastore, mode='mount')

# Step 1: Data ingestion
data_ingestion_step, data_ingestion_outputs = data_ingestion_step(
    datastore, cpu_compute_target)

# Step 2: Data preprocessing
data_preprocess_step, data_preprocess_outputs = data_preprocess_step(
    data_ingestion_outputs['raw_data_dir'], cpu_compute_target)

# Step 3: Train Model
train_step, train_outputs = train_step(data_preprocess_outputs['train_dir'],
                                       data_preprocess_outputs['valid_dir'],
                                       gpu_compute_target)

# Step 4: Evaluate Model
evaluate_step, evaluate_outputs = evaluate_step(
    train_outputs['model_dir'], data_preprocess_outputs['test_dir'],
    gpu_compute_target)

# Step 5: Deploy Model
deploy_step, deploy_outputs = deploy_step(train_outputs['model_dir'],
                                          evaluate_outputs['accuracy_file'],
                                          data_preprocess_outputs['test_dir'],
                                          cpu_compute_target)

# Submit pipeline
print('Submitting pipeline ...')
# Step 1: Data ingestion
ingest_step, ingest_outputs = ingest_step(datastore, cpu_compute_target)

# Step 2: Data preprocessing
preprocess_step, preprocess_outputs = preprocess_step(
    ingest_outputs['raw_data_dir'], cpu_compute_target)

# Step 3: Create vocab from training data
build_vocab_step, build_vocab_outputs = build_vocab_step(
    preprocess_outputs['train_dir'], cpu_compute_target)

# Step 4: Train Model
train_step, train_outputs = train_step(datastore,
                                       preprocess_outputs['train_dir'],
                                       preprocess_outputs['valid_dir'],
                                       build_vocab_outputs['vocab_dir'],
                                       gpu_compute_target)

# Step 5: Evaluate Model
evaluate_step, evaluate_outputs = evaluate_step(datastore,
                                                preprocess_outputs['test_dir'],
                                                train_outputs['model_dir'],
                                                gpu_compute_target)

# Step 6: Deploy Model
deploy_step, deploy_outputs = deploy_step(train_outputs['model_dir'],
                                          evaluate_outputs['eval_dir'],
                                          preprocess_outputs['test_dir'],
                                          cpu_compute_target)