def test_module_with_execute(self):
# delete files created before
result_dir = self.prepare_outputs()['scored_data_output_dir']
os.makedirs(result_dir, exist_ok=True)
if len(os.listdir(result_dir)) > 0:
for file in os.listdir(result_dir):
path = os.path.join(result_dir, file)
os.remove(path)
# This test simulates a parallel run from cmd line arguments to call fasttext_score_parallel.
ModuleExecutor(fasttext_score).execute(self.prepare_argv())
data_dir = self.prepare_inputs()['texts_to_score']
os.makedirs(data_dir, exist_ok=True)
num_of_test_file = len(os.listdir(data_dir))
num_of_test_result = 0
for file in os.listdir(result_dir):
path = os.path.join(result_dir, file)
num = pd.read_parquet(path).shape[0]
num_of_test_result += num
self.assertEqual(num_of_test_file, num_of_test_result)
# for metrics
run.log(name='num of total data', value=len(data))
run.log(name='num of training data', value=len(train))
run.log(name='num of validation data', value=len(dev))
run.log(name='num of test_data', value=len(test))
path_label = os.path.join(input_dir, 'label.txt')
path_word_to_index = os.path.join(input_dir, 'word_to_index.json')
shutil.copy(src=path_label, dst=training_data_output)
shutil.copy(src=path_word_to_index, dst=training_data_output)
path = os.path.join(training_data_output, "data.txt")
with open(path, 'w', encoding='utf-8') as f:
f.writelines(train)
shutil.copy(src=path_label, dst=validation_data_output)
shutil.copy(src=path_word_to_index, dst=validation_data_output)
path = os.path.join(validation_data_output, "data.txt")
with open(path, 'w', encoding='utf-8') as f:
f.writelines(dev)
shutil.copy(src=path_label, dst=test_data_output)
shutil.copy(src=path_word_to_index, dst=test_data_output)
path = os.path.join(test_data_output, "data.txt")
with open(path, 'w', encoding='utf-8') as f:
f.writelines(test)
print('============================================')
if __name__ == '__main__':
ModuleExecutor(split_data_txt).execute(sys.argv)
import shutil
from azureml.pipeline.wrapper.dsl.module import ModuleExecutor, InputDirectory, OutputDirectory
from azureml.pipeline.wrapper import dsl
@dsl.module(name="copy_files")
def copy_files(
output_dir: OutputDirectory(),
input_dir: InputDirectory() = '.',
str_param='some_string',
):
input_dir = Path(input_dir)
print(f'input_dir: {input_dir.resolve()}')
print(f'str_param: {str_param}')
files = []
if input_dir.is_dir():
files = [str(f) for f in input_dir.iterdir()]
if (len(files) == 0):
raise ValueError(f'input_dir should be an directory with files')
output_dir = Path(output_dir)
with open(output_dir / f"output.txt", 'w') as fout:
fout.write(str_param)
if __name__ == '__main__':
ModuleExecutor(copy_files).execute(sys.argv)
if torch.cuda.is_available():
tensor = tensor.cuda()
with torch.no_grad():
output = model(tensor)
softmax = nn.Softmax(dim=1)
pred_probs = softmax(output).cpu().numpy()[0]
index = torch.argmax(output, 1)[0].cpu().item()
result = {
'Filename': Path(f).name,
'Class': MNIST.classes[index]
}
for c, prob in zip(MNIST.classes, pred_probs):
result[f"Prob of {c}"] = prob
results.append(result)
columns = sorted(list(results[0].keys()))
df = pd.DataFrame(results, columns=columns)
print("Result:")
print(df)
output_file = os.path.join(scored_dataset, f"{uuid4().hex}.parquet")
df.to_parquet(output_file, index=False)
return results
return run
# This main code is only used for local debugging, will never be reached in AzureML when it is a parallel module.
# See https://docs.microsoft.com/en-us/azure/machine-learning/how-to-use-parallel-run-step#write-your-inference-script
if __name__ == '__main__':
ModuleExecutor(parallel_score_images).execute(sys.argv)
description='merge two datasets',
name='nyc taxi merge',
)
def merge(
cleaned_yellow_data: InputDirectory(
description=
"cleaned yellow data, needs to be read as pandas dataframe"),
cleaned_green_data: InputDirectory(
description=
"cleaned green data, needs to be read as pandas dataframe"),
merged_output: OutputDirectory(description="output data after merge"),
):
green_df = pd.read_csv(cleaned_green_data)
yellow_df = pd.read_csv(cleaned_yellow_data)
print("Argument (output merge taxi data path): %s" % merged_output)
merge_df = green_df.append(yellow_df, ignore_index=True)
merge_df.reset_index(inplace=True, drop=True)
if not (merged_output is None):
os.makedirs(merged_output, exist_ok=True)
print("merge output folder %s created" % merged_output)
path = merged_output + "/merged.csv"
write_df = merge_df.to_csv(path)
if __name__ == '__main__':
ModuleExecutor(merge).execute(sys.argv)
# for metrics
run.log(name='num of total data', value=len(data))
run.log(name='num of training data', value=len(train))
run.log(name='num of validation data', value=len(dev))
run.log(name='num of test_data', value=len(test))
os.makedirs(training_data_output, exist_ok=True)
path = os.path.join(training_data_output, "train.txt")
with open(path, 'w', encoding='utf-8') as f:
f.writelines(train)
print(path)
print(os.listdir(training_data_output))
os.makedirs(validation_data_output, exist_ok=True)
path = os.path.join(validation_data_output, "dev.txt")
with open(path, 'w', encoding='utf-8') as f:
f.writelines(dev)
print(path)
print(os.listdir(validation_data_output))
os.makedirs(test_data_output, exist_ok=True)
for i, t in enumerate(test):
path_new = os.path.join(test_data_output, str(i))
with open(path_new, 'w', encoding='utf-8') as f:
f.write(t)
print('============================================')
if __name__ == '__main__':
ModuleExecutor(split_data_txt_parallel).execute(sys.argv)
def fasttext_evaluation(
model_testing_result: OutputDirectory(type='AnyDirectory'),
trained_model_dir: InputDirectory(type='AnyDirectory') = None,
test_data_dir: InputDirectory(type='AnyDirectory') = None,
char2index_dir: InputDirectory(type='AnyDirectory') = None):
print('=====================================================')
print(f'trained_model_dir: {Path(trained_model_dir).resolve()}')
print(f'test_data_dir: {Path(test_data_dir).resolve()}')
print(f'char2index_dir: {Path(char2index_dir).resolve()}')
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
max_len_ = 38
path = os.path.join(test_data_dir, 'test.txt')
test_samples = load_dataset(file_path=path,
max_len=max_len_,
char2index_dir=char2index_dir)
test_iter = DataIter(test_samples)
path = os.path.join(trained_model_dir, 'BestModel')
model = torch.load(f=path)
path = os.path.join(model_testing_result, 'result.json')
acc_ = test(model, test_iter, device)
json.dump({"acc": acc_}, open(path, 'w'))
print('\n============================================')
if __name__ == '__main__':
ModuleExecutor(fasttext_evaluation).execute(sys.argv)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print('device:', device)
path = os.path.join(fasttext_model_dir, 'shared_params.json')
with open(path, 'r', encoding='utf-8') as f:
shared_params = json.load(f)
path = os.path.join(fasttext_model_dir, 'BestModel')
model = torch.load(f=path, map_location=device)
def run(files):
if len(files) == 0:
return []
with torch.no_grad():
test_samples = load_dataset(file_path=files, max_len=shared_params['max_len'],
ngram_size=shared_params['ngram_size'], word_to_index=word_to_index,
map_label_id=map_label_id)
test_iter = DataIter(samples=test_samples, batch_size=1, shuffle=False, device=device)
results = predict_parallel(model, test_iter, map_id_label)
dict_ = {'Filename': files, 'Class': results}
df = pd.DataFrame(data=dict_)
output_file = os.path.join(scored_data_output_dir, f"{uuid4().hex}.parquet")
df.to_parquet(output_file, index=False)
return results
return run
# This main code is only used for local debugging, will never be reached in AzureML when it is a parallel module.
# See https://docs.microsoft.com/en-us/azure/machine-learning/how-to-use-parallel-run-step#write-your-inference-script
if __name__ == '__main__':
ModuleExecutor(fasttext_score).execute(sys.argv)
@dsl.module(
description='slice input video into images and audio',
name='slice video',
)
def slice_video(
input_video: InputDirectory(
description="input directory of video file") = './data/input/video',
output_audio: OutputDirectory(
description="output directory of audio from video"
) = '/data/output/video',
output_images: OutputDirectory(
description="output directory of images slice from video"
) = '/data/output/images',
):
## this module takes input video, and slice the video into images with ffmpeg
subprocess.run("ffmpeg -i {} {}/video.aac".format(input_video,
output_audio),
shell=True,
check=True)
subprocess.run("ffmpeg -i {} {}/%05d_video.jpg -hide_banner".format(
input_video, output_images),
shell=True,
check=True)
if __name__ == '__main__':
ModuleExecutor(slice_video).execute(sys.argv)
type: EnumParameter(
enum=EnumType,
description="Whether to use word tokenizer or sentence tokenizer"
) = EnumType.word,
):
sys.argv = [
'tokenizer.py',
'-i',
str(input_file_path),
'-o',
str(output_dir_path),
'--output_to_file',
str(output_to_file),
'--input_is_tsv',
str(input_is_tsv),
'-m',
mode.value,
'-t',
type.value,
]
if delimiter is not None:
sys.argv += ['--delimiter', str(delimiter)]
if ignore_cols is not None:
sys.argv += ['--ignore_cols', str(ignore_cols)]
print(' '.join(sys.argv))
runpy.run_path('tokenizer.py', run_name='__main__')
if __name__ == '__main__':
ModuleExecutor(tokenizer).execute(sys.argv)
if input_dir:
print(f"Contents of input directory:")
print('\n'.join(f.name for f in Path(input_dir).iterdir()))
print(f"Arg 'input_file' = {input_file}, type='{type(input_file)}'")
print(f"Arg 'output_dir' = {output_dir}, type='{type(output_dir)}'")
print(f"Arg 'output_file' = {output_file}, type='{type(output_file)}'")
print(f"Arg 'str_param' = {str_param}, type='{type(str_param)}'")
print(f"Arg 'int_param' = {int_param}, type='{type(int_param)}'")
print(f"Arg 'float_param' = {float_param}, type='{type(float_param)}'")
print(f"Arg 'bool_param' = {bool_param}, type='{type(bool_param)}'")
print(f"Arg 'enum_param' = {enum_param}, type='{type(enum_param)}'")
print(f"Arg 'none_param' = {none_param}, type='{type(none_param)}'")
data = str_param
if input_file:
with open(input_file, 'r') as fin:
data = fin.read()
print("Content of input file:", data)
if input_dir:
shutil.copytree(input_dir, output_dir)
else:
os.makedirs(output_dir, exist_ok=True)
with open(os.path.join(output_dir, "test.txt"), 'w') as fout:
fout.write(data)
with open(output_file, 'w') as fout:
fout.write(data)
if __name__ == '__main__':
ModuleExecutor(sample_module).execute(sys.argv)
val2 = 'val2'
@dsl.module(
description='A sample module which shows the input data.',
name='Prepare data',
)
def prepare_data(
output_data: OutputDirectory(),
input_data: InputDirectory() = None,
str_param: str = None,
int_param: int = 0,
enum_param: EnumEnumParam = None,
):
sys.argv = [
'prepare_data.py',
'--input_data', str(input_data),
'--output_data', str(output_data),
'--int_param', str(int_param),
]
if str_param is not None:
sys.argv += ['--str_param', str(str_param)]
if enum_param is not None:
sys.argv += ['--enum_param', enum_param.value]
print(' '.join(sys.argv))
runpy.run_path('prepare_data.py', run_name='__main__')
if __name__ == '__main__':
ModuleExecutor(prepare_data).execute(sys.argv)
print(f'char2index_dir: {Path(char2index_dir).resolve()}')
print(f'scored_dataset: {scored_dataset}')
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
max_len_ = 38
path = os.path.join(fasttext_model, 'BestModel')
model = torch.load(f=path)
def run(files):
if len(files) == 0:
return []
print(f"Ready to process {len(files)} texts.")
print('\n'.join(files))
with torch.no_grad():
test_samples = load_dataset_parallel(files=files, max_len=max_len_, char2index_dir=char2index_dir)
test_iter = DataIter_Parallel(test_samples, shuffle=False)
results = predict_parallel(model, test_iter, device)
dict_ = {'Filename': files, 'Class': results}
df = pd.DataFrame(data=dict_)
print("Result:")
print(df)
output_file = os.path.join(scored_dataset, f"{uuid4().hex}.parquet")
df.to_parquet(output_file, index=False)
return results
return run
if __name__ == '__main__':
ModuleExecutor(fasttext_score_parallel).execute(sys.argv)
# ---------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# ---------------------------------------------------------
import sys
from azureml.pipeline.wrapper.dsl.module import ModuleExecutor
from azureml.pipeline.wrapper import dsl
from package1.foo import bar
@dsl.module(name="Basic Module")
def basic_module(
string_parameter: str,
int_parameter: int,
boolean_parameter: bool,
string_parameter_with_default='abc',
):
bar()
if __name__ == '__main__':
ModuleExecutor(basic_module).execute(sys.argv)
# ---------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# ---------------------------------------------------------
import sys
from azureml.pipeline.wrapper import dsl
from azureml.pipeline.wrapper.dsl.module import ModuleExecutor, OutputFile
from my_module import *
import my_module
@dsl.module()
def add(
output_file: OutputFile(),
a=1,
b=2,
c=3,
):
return my_module.add(**locals())
if __name__ == '__main__':
ModuleExecutor(add).execute(sys.argv)
import gdal
from azureml.pipeline.wrapper.dsl.module import ModuleExecutor, InputDirectory, OutputDirectory
from azureml.studio.core.io.data_frame_directory import load_data_frame_from_directory
from azureml.pipeline.wrapper import dsl
gdal_version_num = int(gdal.VersionInfo('VERSION_NUM'))
print(f'gdal version number is {gdal_version_num}.')
@dsl.module(name="gdal_sample")
def gdal_sample(
##define interface(input, output, paratmers) of the module here
output_dir1: OutputDirectory(),
output_dir2: OutputDirectory(),
input_dir1: InputDirectory(),
input_dir2: InputDirectory()):
print('I am in module definition')
print(f'input_dir1: {Path(input_dir1).resolve()}')
print(f'input_dir2: {Path(input_dir2).resolve()}')
## add custom logic here
dfd1 = load_data_frame_from_directory(input_dir1)
data_frame1 = dfd1.data
print(data_frame1.head(10))
if __name__ == '__main__':
ModuleExecutor(gdal_sample).execute(sys.argv)
import sys
from pathlib import Path
from tempfile import mkdtemp
from azureml.pipeline.wrapper.dsl.module import ModuleExecutor, InputDirectory, OutputDirectory
from azureml.pipeline.wrapper import dsl
from azureml.core.run import Run
@dsl.module(
name='enter_num_manually',
description='Put a number in parameter and this module will convert it to a file',
job_type='basic')
def enter_num_manually(output: OutputDirectory(), num='0'):
Path(output).absolute().mkdir(parents=True, exist_ok=True)
with open(Path(output).resolve() / f'data', 'w') as fout:
fout.write(num)
if __name__ == "__main__":
ModuleExecutor(enter_num_manually).execute(sys.argv)
import sys
from pathlib import Path
from tempfile import mkdtemp
from azureml.pipeline.wrapper.dsl.module import ModuleExecutor
from azureml.pipeline.wrapper import dsl
from azureml.core.run import Run
@dsl.module(job_type="basic", name="multiply")
def multiply(left: Path = Path('.'), right: Path = Path('.')) -> Path:
l = float(Path(left.resolve() / f'data').read_text().strip())
r = float(Path(right.resolve() / f'data').read_text().strip())
print('left = ', l)
print('right = ', r)
result = l * r
run = Run.get_context()
run.log('result', result)
run.flush()
output_path = Path(mkdtemp())
with open(output_path / f'data', 'w') as fout:
fout.write(str(result))
return output_path
if __name__ == "__main__":
ModuleExecutor(multiply).execute(sys.argv)
):
print('=====================================================')
print(f'input_dir: {Path(first_trained_model).resolve()}')
print(f'input_dir: {Path(first_trained_result).resolve()}')
print(f'input_dir: {Path(second_trained_model).resolve()}')
print(f'input_dir: {Path(second_trained_result).resolve()}')
# for logging
run = Run.get_context()
path = os.path.join(first_trained_result, 'result.json')
result_first = json.load(open(path, 'r'))['acc']
path = os.path.join(second_trained_result, 'result.json')
second_first = json.load(open(path, 'r'))['acc']
dst = os.path.join(the_better_model, 'BestModel')
if result_first >= second_first:
print('choose the first model')
run.log(name='which one', value='first')
src = os.path.join(first_trained_model, 'BestModel')
shutil.copy(src=src, dst=dst)
else:
print('choose the second model')
run.log(name='which one', value='second')
src = os.path.join(second_trained_model, 'BestModel')
shutil.copy(src=src, dst=dst)
print('=====================================================')
if __name__ == '__main__':
ModuleExecutor(compare_two_models).execute(sys.argv)
@dsl.module(
description='stitch images and audio back to video',
name='stitch video',
)
def stitch_video(
input_images: InputDirectory(description="input directory of images"),
input_audio: InputDirectory(description="input directory of audio"),
output_video: OutputDirectory(
description="output directory of stitched video file")):
## this module takes input video, and slice the video into images with ffmpeg
subprocess.run(
"ffmpeg -framerate 30 -i {}/%05d_video.jpg -c:v libx264 -profile:v high -crf 20 -pix_fmt yuv420p "
"-y {}/video_without_audio.mp4".format(args.images_dir,
args.output_dir),
shell=True,
check=True)
subprocess.run(
"ffmpeg -i {}/video_without_audio.mp4 -i {}/video.aac -map 0:0 -map 1:0 -vcodec "
"copy -acodec copy -y {}/video_with_audio.mp4".format(
args.output_dir, args.input_audio, args.output_dir),
shell=True,
check=True)
if __name__ == '__main__':
ModuleExecutor(stitch_video).execute(sys.argv)
def split_samples(self, sub_samples):
b_x = [item[0] for item in sub_samples]
b_y = [item[1] for item in sub_samples]
return np.array(b_x), np.array(b_y)
def __next__(self):
if (self.index == self.n_batches) and (self.residue is True):
sub_samples = self.samples[self.index * self.batch_size: len(self.samples)]
self.index += 1
return self.split_samples(sub_samples)
elif self.index >= self.n_batches:
self.index = 0
raise StopIteration
else:
sub_samples = self.samples[self.index * self.batch_size: (self.index + 1) * self.batch_size]
self.index += 1
return self.split_samples(sub_samples)
def __iter__(self):
return self
def __len__(self):
if self.residue:
return self.n_batches + 1
else:
return self.n_batches
if __name__ == '__main__':
ModuleExecutor(fasttext_test).execute(sys.argv)
from azureml.pipeline.wrapper import dsl
from azureml.pipeline.wrapper.dsl.module import ModuleExecutor, InputDirectory, OutputDirectory
@dsl.module(name="MPI Module", job_type='mpi')
def mpi_module(
output_dir: OutputDirectory(),
input_dir: InputDirectory() = '.',
param0: str = 'abc',
param1: int = 10,
):
from mpi4py import MPI
for k, v in locals().items():
print(f"{k}: {v}")
comm = MPI.COMM_WORLD
size = comm.Get_size()
rank = comm.Get_rank()
print(f"This is an MPI module, I'm rank {rank}/{size}.")
if rank == 0:
print("I will write data.")
output_dir = Path(output_dir)
with open(output_dir / f"output.txt", 'w') as fout:
fout.write(param0)
fout.write(str(param1))
else:
print("I don't return data.")
if __name__ == '__main__':
ModuleExecutor(mpi_module).execute(sys.argv)
def test_module_with_execute(self):
# This test simulates a parallel run from cmd line arguments to call parallel_sample.
ModuleExecutor(parallel_score_images).execute(self.prepare_argv())
max_len_ = 38
n_class_ = len(class_)
vocab_size_ = len(c2i)
stop_patience = 5
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
path = os.path.join(training_data_dir, 'train.txt')
train_samples = load_dataset(file_path=path, max_len=max_len_, char2index_dir=char2index_dir)
path = os.path.join(validation_data_dir, 'dev.txt')
dev_samples = load_dataset(file_path=path, max_len=max_len_, char2index_dir=char2index_dir)
train_iter = DataIter(train_samples, batch_size)
dev_iter = DataIter(dev_samples, batch_size)
model = FastText(vocab_size=vocab_size_, n_class=n_class_, embed_dim=embedding_dim)
start = time.time()
train(model,
trained_model_dir,
train_iter,
dev_iter=dev_iter,
epochs=epochs,
learning_rate=learning_rate,
stop_patience=stop_patience,
device=device)
end = time.time()
print('\nspent time: %.2f sec' % (end - start))
print('============================================')
if __name__ == '__main__':
ModuleExecutor(fasttext_train).execute(sys.argv)
style_model.load_state_dict(state_dict)
style_model.to(device)
print(f'Model loaded successfully. Path: {model_dir}')
def run(mini_batch):
result = []
for image_file_path in mini_batch:
img = load_image(image_file_path)
print(f'load image from: {image_file_path}')
with torch.no_grad():
content_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Lambda(lambda x: x.mul(255))
])
content_image = content_transform(img)
content_image = content_image.unsqueeze(0).to(device)
output = style_model(content_image).cpu()
output_file_path = os.path.join(
output_path, os.path.basename(image_file_path))
save_image(output_file_path, output[0])
result.append(output_file_path)
print(f'transferred image saved in: {output_file_path}')
return result
return run
# This main code is only used for local debugging, will never be reached in AzureML when it is a parallel module.
if __name__ == '__main__':
ModuleExecutor(style_transform_parallel).execute(sys.argv)