def list_files(self, bucket_name, prefix="", *args, **kwargs):
"""
Lists files in file storage
:param bucket_name: name of the bucket/container
:param prefix: prefix string to search by
:param args: other arguments containing additional information
:param kwargs: other keyword arguments containing additional information
"""
try:
# get run_id from object_name
current_id = prefix
if current_id not in self.downloaded_files.keys():
# load registered models
model_name = get_model_register_name(current_id)
model = Model(self.ws, name=model_name)
model.download(target_dir=current_id, exist_ok=True)
# mark current_id as registered
self.downloaded_files[current_id] = {"path": current_id}
# get files from current_id folder and temporary_storage folder
list_files = os.listdir(os.path.join(current_id, AML_MLAPP_FOLDER))
list_files_extra = list(
map(
lambda f: os.path.basename(f),
glob.glob(
os.path.join(self.temporary_storage,
current_id + "_*"))))
list_files.extend(list_files_extra)
return list_files
except Exception as e:
logging.error(e)
raise e
def download_file(self, bucket_name, object_name, file_path, *args,
**kwargs):
"""
This method downloads a file into your machine from AML datastore
:param bucket_name: name of the bucket/container, unnecessary parameter
:param object_name: name of the object/file
:param file_path: path to local file, unnecessary parameter
:param args: other arguments containing additional information
:param kwargs: other keyword arguments containing additional information
:return: None
"""
# get run_id from object_name
current_id = object_name.split('_')[0]
if current_id not in self.downloaded_files.keys(
) or not self.downloaded_files[current_id]:
# load registered models
model_name = get_model_register_name(current_id)
model = Model(self.ws, name=model_name)
model.download(target_dir=current_id, exist_ok=True)
# mark current_id as registered
self.downloaded_files[current_id] = {"path": current_id}
# move file to AML_MLAPP_FOLDER folder from current_id folder
if os.path.exists(
os.path.join(current_id, AML_MLAPP_FOLDER, object_name)):
shutil.move(
os.path.join(current_id, AML_MLAPP_FOLDER, object_name),
self.temporary_storage)
def run(mini_batch):
print(f'run method start: {__file__}, run({mini_batch})')
timestamp_column = 'WeekStarting'
timeseries_id_columns = ['Store', 'Brand']
data = mini_batch \
.set_index(timestamp_column) \
.sort_index(ascending=True)
#Prepare loading model from Azure ML, get the latest model by default
model_name = "prs_" + str(data['Store'].iloc[0]) + "_" + str(
data['Brand'].iloc[0])
model = Model(ws, model_name)
model.download(exist_ok=True)
forecaster = joblib.load(model_name)
# Get predictions
#This is to append the store and brand column to the result
ts_id_dict = {
id_col: str(data[id_col].iloc[0])
for id_col in timeseries_id_columns
}
forecasts = forecaster.forecast(data)
prediction_df = forecasts.to_frame(name='Prediction')
prediction_df = prediction_df.reset_index().assign(**ts_id_dict)
return prediction_df
def run(raw_data):
Inputs = pd.DataFrame(ast.literal_eval(json.loads(raw_data)['Inputs']))
timestamp_column = 'WeekStarting'
Inputs[timestamp_column] = pd.to_datetime(Inputs[timestamp_column])
timeseries_id_columns = ['Store', 'Brand']
data = Inputs \
.set_index(timestamp_column) \
.sort_index(ascending=True)
#Prepare loading model from Azure ML, get the latest model by default
model_name = "prs_" + str(data['Store'].iloc[0]) + "_" + str(
data['Brand'].iloc[0])
model = Model(ws, model_name)
model.download(exist_ok=True)
forecaster = joblib.load(model_name)
# Get predictions
#This is to append the store and brand column to the result
ts_id_dict = {
id_col: str(data[id_col].iloc[0])
for id_col in timeseries_id_columns
}
forecasts = forecaster.forecast(data)
prediction_df = forecasts.to_frame(name='Prediction')
prediction_df = prediction_df.reset_index().assign(**ts_id_dict)
return prediction_df.to_json()
def download_model(ws: Workspace, config: MoveModelConfig) -> Model:
"""
Downloads an InnerEye model from an AzureML workspace
:param ws: The AzureML workspace
:param config: move config
:return: the exported Model
"""
model = Model(ws, id=config.model_id)
model_path, environment_path = config.get_paths()
with open(model_path / MODEL_JSON, 'w') as f:
json.dump(model.serialize(), f)
model.download(target_dir=str(model_path))
env_name = model.tags.get(PYTHON_ENVIRONMENT_NAME)
environment = ws.environments.get(env_name)
environment.save_to_directory(str(environment_path), overwrite=True)
return model
def load_model_from_registry(model_name, model_version):
logging.info(
f' >> load_model_from_registry({model_name},{model_version}) ...')
try:
aml_model = Model(workspace, name=model_name, version=model_version)
model_path = aml_model.download(target_dir='.', exist_ok=True)
return joblib.load(model_path)
except Exception as e:
logging.error(e)
raise
def get_checkpoints_from_model(model_id: str, workspace: Workspace, download_path: Path) -> List[Path]:
if len(model_id.split(":")) != 2:
raise ValueError(
f"model_id should be in the form 'model_name:version', got {model_id}")
model_name, model_version = model_id.split(":")
model = Model(workspace=workspace, name=model_name, version=int(model_version))
model_path = Path(model.download(str(download_path), exist_ok=True))
model_inference_config = read_model_inference_config(model_path / MODEL_INFERENCE_JSON_FILE_NAME)
checkpoint_paths = [model_path / x for x in model_inference_config.checkpoint_paths]
return checkpoint_paths
def get_or_create_model(self):
"""
Get or Create Model
:return: Model from Workspace
"""
assert self.model_name
if self.model_name in self.models:
# if get_model(self.model_name).tags['train_py_hash'] == self.get_file_md5(
# self.source_directory + "/" + self.script):
model = Model(self, name=self.model_name)
model.download("outputs", exist_ok=True)
return model
model = self.train_model()
assert model
if self.show_output:
print(model.name, model.version, model.url, sep="\n")
return model
def run(project_root: Optional[Path] = None) -> None:
"""
Runs inference on an image. This can be invoked in one of two ways:
(1) when there is already a model in the project_root directory; this is the case when
we arrive here from python_wrapper.py
(2) when we need to download a model, which must be specified by the --model-id switch.
This is the case when this script is invoked by submit_for_inference.py.
:param project_root: the directory in which the model (including code) is located.
Must be None if and only if the --model-id switch is provided.
"""
parser = argparse.ArgumentParser(
description=
'Execute code baked into a Docker Container from AzureML ScriptRunConfig'
)
parser.add_argument('--spawnprocess',
dest='spawnprocess',
action='store',
type=str)
parser.add_argument('--data-folder',
dest='data_folder',
action='store',
type=str)
parser.add_argument('--model-id',
dest='model_id',
action='store',
type=str)
known_args, unknown_args = parser.parse_known_args()
if known_args.model_id:
if project_root:
raise ValueError(
"--model-id should not be provided when project_root is specified"
)
workspace = Run.get_context().experiment.workspace
model = Model(workspace=workspace, id=known_args.model_id)
current_dir = Path(".")
project_root = Path(model.download(str(current_dir))).absolute()
elif not project_root:
raise ValueError(
"--model-id must be provided when project_root is unspecified")
script_path = Path('run_score.sh')
write_script(parser, script_path, project_root)
print(f"Running {script_path} ...")
env = dict(os.environ.items())
# Work around https://github.com/pytorch/pytorch/issues/37377
env['MKL_SERVICE_FORCE_INTEL'] = '1'
code = spawn_and_monitor_subprocess(process='bash',
args=[str(script_path)],
env=env)
sys.exit(code)
def get_or_create_model(self) -> Model:
"""
Get or Create Model
:return: Model from Workspace
"""
assert self.model_name
print("Check if Model exists.")
if self.model_name in self.models:
print("Model does exists.")
# if get_model(self.model_name).tags['train_py_hash'] == self.get_file_md5(
# self.source_directory + "/" + self.script):
model = Model(self, name=self.model_name)
if not os.path.isdir("outputs"):
model.download("outputs", exist_ok=True)
return model
print("Model does not exists.")
model = self.train_model()
assert model
if self.show_output:
print(model.name, model.version, model.url, sep="\n")
return model
def run() -> None:
"""
Downloads a model from AzureML, and starts the score script (usually score.py) in the root folder of the model.
Downloading the model is only supported if the present code is running inside of AzureML. When running outside
of AzureML, the model must have been downloaded beforehand into the folder given by the model-folder argument.
The script is executed with the current Python interpreter.
If the model requires a specific Conda environment to run in, the caller of this script needs to ensure
that this has been set up correctly (taking the environment.yml file stored in the model).
All arguments that are not recognized by the present code will be passed through to `score.py` unmodified.
Example arguments:
download_model_and_run_scoring.py --model-id=Foo:1 score.py --foo=1 --bar
This would attempt to download version 1 of model Foo, and then start the script score.py in the model's root
folder. Arguments --foo and --bar are passed through to score.py
"""
parser = argparse.ArgumentParser(description='Execute code inside of an AzureML model')
# Use argument names with dashes here. The rest of the codebase uses _ as the separator, meaning that there
# can't be a clash of names with arguments that are passed through to score.py
parser.add_argument('--model-folder', dest='model_folder', action='store', type=str)
parser.add_argument('--model-id', dest='model_id', action='store', type=str)
known_args, unknown_args = parser.parse_known_args()
model_folder = known_args.model_folder or "."
if known_args.model_id:
current_run = Run.get_context()
if not hasattr(current_run, 'experiment'):
raise ValueError("The model-id argument can only be used inside AzureML. Please drop the argument, and "
"supply the downloaded model in the model-folder.")
workspace = current_run.experiment.workspace
model = Model(workspace=workspace, id=known_args.model_id)
# Download the model from AzureML into a sub-folder of model_folder
model_folder = str(Path(model.download(model_folder)).absolute())
env = dict(os.environ.items())
# Work around https://github.com/pytorch/pytorch/issues/37377
env['MKL_SERVICE_FORCE_INTEL'] = '1'
# The model should include all necessary code, hence point the Python path to its root folder.
env['PYTHONPATH'] = model_folder
if not unknown_args:
raise ValueError("No arguments specified for starting the scoring script.")
score_script = Path(model_folder) / unknown_args[0]
score_args = [str(score_script), *unknown_args[1:]]
if not score_script.exists():
raise ValueError(f"The specified entry script {score_args[0]} does not exist in {model_folder}")
print(f"Starting Python with these arguments: {' '.join(score_args)}")
code, stdout = spawn_and_monitor_subprocess(process=sys.executable, args=score_args, env=env)
if code != 0:
print(f"Python terminated with exit code {code}. Stdout: {os.linesep.join(stdout)}")
sys.exit(code)
# Parse command line arguments
argparser = argparse.ArgumentParser()
argparser.add_argument("--modelname", type=str, default="char_rnn_model")
argparser.add_argument("-p", "--prime_str", type=str, default="A")
argparser.add_argument("-l", "--predict_len", type=int, default=100)
argparser.add_argument("-t", "--temperature", type=float, default=0.8)
argparser.add_argument("--cuda", action="store_true")
args = argparser.parse_args()
all_characters = string.printable
n_characters = len(all_characters)
# TODO: Use the Model class to download your trained model
model = Model(workspace=ws, name=args.modelname)
model = model.download(target_dir='.', exist_ok=True)
# Loading the model and reconstructs the model
filename = "outputs/" + args.modelname + ".pt"
decoder_state = torch.load(filename)
char_rnn = CharRNN(n_characters, 100, n_characters, "gru", 2)
char_rnn.load_state_dict(decoder_state)
# Generate the predicted sequence based of the hyperparameters
predicted = generate(decoder=char_rnn,
prime_str=args.prime_str,
predict_len=args.predict_len,
temperature=args.temperature,
cuda=args.cuda)
print(predicted)
import os
from azureml.core import Workspace, Model
from azureml.core.authentication import AzureCliAuthentication
ws = Workspace.from_config(
auth=AzureCliAuthentication()
)
model = Model(
workspace=ws,
name="net.onnx"
)
model.download(
target_dir=os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'../',
'outputs'
)
)
print(model)
def load_champion_model(model_name: str) -> Any:
champ = Model(workspace, model_name)
champ_model_path = champ.download(target_dir='./champion')
return joblib.load(champ_model_path)
import azureml.core
from azureml.core import Model, Workspace
model_name = 'gensim_lda'
print("Azure ML SDK Version: ", azureml.core.VERSION)
ws = Workspace.from_config()
print(ws.name, ws.location, ws.resource_group, sep='\t')
model = Model(workspace=ws, name=model_name)
model.download()
# download model
import json
from azureml.core import Workspace, Model, VERSION
from azureml.core.authentication import ServicePrincipalAuthentication
print(VERSION)
with open("aml/config.json", "r") as f:
config = json.load(f)
auth = ServicePrincipalAuthentication(config["tenant_id"],
config["service_principal_id"],
config["service_principal_password"])
ws = Workspace.create(
name=config["workspace_name"],
auth=auth,
subscription_id=config['subscription_id'],
resource_group=config['resource_group'],
exist_ok=True,
show_output=True,
)
model = Model(ws, name="TinyYOLO")
model.download(target_dir="modules/InferenceModule/")
