gpu_compute_target.wait_for_completion(show_output=True)
# Get datastore reference
datastore = DataReference(datastore, mode='mount')
# Step 1: Data ingestion
data_ingestion_step, data_ingestion_outputs = data_ingestion_step(
datastore, cpu_compute_target)
# Step 2: Data preprocessing
data_preprocess_step, data_preprocess_outputs = data_preprocess_step(
data_ingestion_outputs['raw_data_dir'], cpu_compute_target)
# Step 3: Train Model
train_step, train_outputs = train_step(data_preprocess_outputs['train_dir'],
data_preprocess_outputs['valid_dir'],
gpu_compute_target)
# Step 4: Evaluate Model
evaluate_step, evaluate_outputs = evaluate_step(
train_outputs['model_dir'], data_preprocess_outputs['test_dir'],
gpu_compute_target)
# Step 5: Deploy Model
deploy_step, deploy_outputs = deploy_step(train_outputs['model_dir'],
evaluate_outputs['accuracy_file'],
data_preprocess_outputs['test_dir'],
cpu_compute_target)
# Submit pipeline
print('Submitting pipeline ...')
# Step 1: Data ingestion
ingest_step, ingest_outputs = ingest_step(datastore, cpu_compute_target)
# Step 2: Data preprocessing
preprocess_step, preprocess_outputs = preprocess_step(
ingest_outputs['raw_data_dir'], cpu_compute_target)
# Step 3: Create vocab from training data
build_vocab_step, build_vocab_outputs = build_vocab_step(
preprocess_outputs['train_dir'], cpu_compute_target)
# Step 4: Train Model
train_step, train_outputs = train_step(datastore,
preprocess_outputs['train_dir'],
preprocess_outputs['valid_dir'],
build_vocab_outputs['vocab_dir'],
gpu_compute_target)
# Step 5: Evaluate Model
evaluate_step, evaluate_outputs = evaluate_step(datastore,
preprocess_outputs['test_dir'],
train_outputs['model_dir'],
gpu_compute_target)
# Step 6: Deploy Model
deploy_step, deploy_outputs = deploy_step(train_outputs['model_dir'],
evaluate_outputs['eval_dir'],
preprocess_outputs['test_dir'],
cpu_compute_target)