def list_files(self, bucket_name, prefix="", *args, **kwargs): """ Lists files in file storage :param bucket_name: name of the bucket/container :param prefix: prefix string to search by :param args: other arguments containing additional information :param kwargs: other keyword arguments containing additional information """ try: # get run_id from object_name current_id = prefix if current_id not in self.downloaded_files.keys(): # load registered models model_name = get_model_register_name(current_id) model = Model(self.ws, name=model_name) model.download(target_dir=current_id, exist_ok=True) # mark current_id as registered self.downloaded_files[current_id] = {"path": current_id} # get files from current_id folder and temporary_storage folder list_files = os.listdir(os.path.join(current_id, AML_MLAPP_FOLDER)) list_files_extra = list( map( lambda f: os.path.basename(f), glob.glob( os.path.join(self.temporary_storage, current_id + "_*")))) list_files.extend(list_files_extra) return list_files except Exception as e: logging.error(e) raise e
def download_file(self, bucket_name, object_name, file_path, *args, **kwargs): """ This method downloads a file into your machine from AML datastore :param bucket_name: name of the bucket/container, unnecessary parameter :param object_name: name of the object/file :param file_path: path to local file, unnecessary parameter :param args: other arguments containing additional information :param kwargs: other keyword arguments containing additional information :return: None """ # get run_id from object_name current_id = object_name.split('_')[0] if current_id not in self.downloaded_files.keys( ) or not self.downloaded_files[current_id]: # load registered models model_name = get_model_register_name(current_id) model = Model(self.ws, name=model_name) model.download(target_dir=current_id, exist_ok=True) # mark current_id as registered self.downloaded_files[current_id] = {"path": current_id} # move file to AML_MLAPP_FOLDER folder from current_id folder if os.path.exists( os.path.join(current_id, AML_MLAPP_FOLDER, object_name)): shutil.move( os.path.join(current_id, AML_MLAPP_FOLDER, object_name), self.temporary_storage)
def run(mini_batch): print(f'run method start: {__file__}, run({mini_batch})') timestamp_column = 'WeekStarting' timeseries_id_columns = ['Store', 'Brand'] data = mini_batch \ .set_index(timestamp_column) \ .sort_index(ascending=True) #Prepare loading model from Azure ML, get the latest model by default model_name = "prs_" + str(data['Store'].iloc[0]) + "_" + str( data['Brand'].iloc[0]) model = Model(ws, model_name) model.download(exist_ok=True) forecaster = joblib.load(model_name) # Get predictions #This is to append the store and brand column to the result ts_id_dict = { id_col: str(data[id_col].iloc[0]) for id_col in timeseries_id_columns } forecasts = forecaster.forecast(data) prediction_df = forecasts.to_frame(name='Prediction') prediction_df = prediction_df.reset_index().assign(**ts_id_dict) return prediction_df
def run(raw_data): Inputs = pd.DataFrame(ast.literal_eval(json.loads(raw_data)['Inputs'])) timestamp_column = 'WeekStarting' Inputs[timestamp_column] = pd.to_datetime(Inputs[timestamp_column]) timeseries_id_columns = ['Store', 'Brand'] data = Inputs \ .set_index(timestamp_column) \ .sort_index(ascending=True) #Prepare loading model from Azure ML, get the latest model by default model_name = "prs_" + str(data['Store'].iloc[0]) + "_" + str( data['Brand'].iloc[0]) model = Model(ws, model_name) model.download(exist_ok=True) forecaster = joblib.load(model_name) # Get predictions #This is to append the store and brand column to the result ts_id_dict = { id_col: str(data[id_col].iloc[0]) for id_col in timeseries_id_columns } forecasts = forecaster.forecast(data) prediction_df = forecasts.to_frame(name='Prediction') prediction_df = prediction_df.reset_index().assign(**ts_id_dict) return prediction_df.to_json()
def download_model(ws: Workspace, config: MoveModelConfig) -> Model: """ Downloads an InnerEye model from an AzureML workspace :param ws: The AzureML workspace :param config: move config :return: the exported Model """ model = Model(ws, id=config.model_id) model_path, environment_path = config.get_paths() with open(model_path / MODEL_JSON, 'w') as f: json.dump(model.serialize(), f) model.download(target_dir=str(model_path)) env_name = model.tags.get(PYTHON_ENVIRONMENT_NAME) environment = ws.environments.get(env_name) environment.save_to_directory(str(environment_path), overwrite=True) return model
def load_model_from_registry(model_name, model_version): logging.info( f' >> load_model_from_registry({model_name},{model_version}) ...') try: aml_model = Model(workspace, name=model_name, version=model_version) model_path = aml_model.download(target_dir='.', exist_ok=True) return joblib.load(model_path) except Exception as e: logging.error(e) raise
def get_checkpoints_from_model(model_id: str, workspace: Workspace, download_path: Path) -> List[Path]: if len(model_id.split(":")) != 2: raise ValueError( f"model_id should be in the form 'model_name:version', got {model_id}") model_name, model_version = model_id.split(":") model = Model(workspace=workspace, name=model_name, version=int(model_version)) model_path = Path(model.download(str(download_path), exist_ok=True)) model_inference_config = read_model_inference_config(model_path / MODEL_INFERENCE_JSON_FILE_NAME) checkpoint_paths = [model_path / x for x in model_inference_config.checkpoint_paths] return checkpoint_paths
def get_or_create_model(self): """ Get or Create Model :return: Model from Workspace """ assert self.model_name if self.model_name in self.models: # if get_model(self.model_name).tags['train_py_hash'] == self.get_file_md5( # self.source_directory + "/" + self.script): model = Model(self, name=self.model_name) model.download("outputs", exist_ok=True) return model model = self.train_model() assert model if self.show_output: print(model.name, model.version, model.url, sep="\n") return model
def run(project_root: Optional[Path] = None) -> None: """ Runs inference on an image. This can be invoked in one of two ways: (1) when there is already a model in the project_root directory; this is the case when we arrive here from python_wrapper.py (2) when we need to download a model, which must be specified by the --model-id switch. This is the case when this script is invoked by submit_for_inference.py. :param project_root: the directory in which the model (including code) is located. Must be None if and only if the --model-id switch is provided. """ parser = argparse.ArgumentParser( description= 'Execute code baked into a Docker Container from AzureML ScriptRunConfig' ) parser.add_argument('--spawnprocess', dest='spawnprocess', action='store', type=str) parser.add_argument('--data-folder', dest='data_folder', action='store', type=str) parser.add_argument('--model-id', dest='model_id', action='store', type=str) known_args, unknown_args = parser.parse_known_args() if known_args.model_id: if project_root: raise ValueError( "--model-id should not be provided when project_root is specified" ) workspace = Run.get_context().experiment.workspace model = Model(workspace=workspace, id=known_args.model_id) current_dir = Path(".") project_root = Path(model.download(str(current_dir))).absolute() elif not project_root: raise ValueError( "--model-id must be provided when project_root is unspecified") script_path = Path('run_score.sh') write_script(parser, script_path, project_root) print(f"Running {script_path} ...") env = dict(os.environ.items()) # Work around https://github.com/pytorch/pytorch/issues/37377 env['MKL_SERVICE_FORCE_INTEL'] = '1' code = spawn_and_monitor_subprocess(process='bash', args=[str(script_path)], env=env) sys.exit(code)
def get_or_create_model(self) -> Model: """ Get or Create Model :return: Model from Workspace """ assert self.model_name print("Check if Model exists.") if self.model_name in self.models: print("Model does exists.") # if get_model(self.model_name).tags['train_py_hash'] == self.get_file_md5( # self.source_directory + "/" + self.script): model = Model(self, name=self.model_name) if not os.path.isdir("outputs"): model.download("outputs", exist_ok=True) return model print("Model does not exists.") model = self.train_model() assert model if self.show_output: print(model.name, model.version, model.url, sep="\n") return model
def run() -> None: """ Downloads a model from AzureML, and starts the score script (usually score.py) in the root folder of the model. Downloading the model is only supported if the present code is running inside of AzureML. When running outside of AzureML, the model must have been downloaded beforehand into the folder given by the model-folder argument. The script is executed with the current Python interpreter. If the model requires a specific Conda environment to run in, the caller of this script needs to ensure that this has been set up correctly (taking the environment.yml file stored in the model). All arguments that are not recognized by the present code will be passed through to `score.py` unmodified. Example arguments: download_model_and_run_scoring.py --model-id=Foo:1 score.py --foo=1 --bar This would attempt to download version 1 of model Foo, and then start the script score.py in the model's root folder. Arguments --foo and --bar are passed through to score.py """ parser = argparse.ArgumentParser(description='Execute code inside of an AzureML model') # Use argument names with dashes here. The rest of the codebase uses _ as the separator, meaning that there # can't be a clash of names with arguments that are passed through to score.py parser.add_argument('--model-folder', dest='model_folder', action='store', type=str) parser.add_argument('--model-id', dest='model_id', action='store', type=str) known_args, unknown_args = parser.parse_known_args() model_folder = known_args.model_folder or "." if known_args.model_id: current_run = Run.get_context() if not hasattr(current_run, 'experiment'): raise ValueError("The model-id argument can only be used inside AzureML. Please drop the argument, and " "supply the downloaded model in the model-folder.") workspace = current_run.experiment.workspace model = Model(workspace=workspace, id=known_args.model_id) # Download the model from AzureML into a sub-folder of model_folder model_folder = str(Path(model.download(model_folder)).absolute()) env = dict(os.environ.items()) # Work around https://github.com/pytorch/pytorch/issues/37377 env['MKL_SERVICE_FORCE_INTEL'] = '1' # The model should include all necessary code, hence point the Python path to its root folder. env['PYTHONPATH'] = model_folder if not unknown_args: raise ValueError("No arguments specified for starting the scoring script.") score_script = Path(model_folder) / unknown_args[0] score_args = [str(score_script), *unknown_args[1:]] if not score_script.exists(): raise ValueError(f"The specified entry script {score_args[0]} does not exist in {model_folder}") print(f"Starting Python with these arguments: {' '.join(score_args)}") code, stdout = spawn_and_monitor_subprocess(process=sys.executable, args=score_args, env=env) if code != 0: print(f"Python terminated with exit code {code}. Stdout: {os.linesep.join(stdout)}") sys.exit(code)
# Parse command line arguments argparser = argparse.ArgumentParser() argparser.add_argument("--modelname", type=str, default="char_rnn_model") argparser.add_argument("-p", "--prime_str", type=str, default="A") argparser.add_argument("-l", "--predict_len", type=int, default=100) argparser.add_argument("-t", "--temperature", type=float, default=0.8) argparser.add_argument("--cuda", action="store_true") args = argparser.parse_args() all_characters = string.printable n_characters = len(all_characters) # TODO: Use the Model class to download your trained model model = Model(workspace=ws, name=args.modelname) model = model.download(target_dir='.', exist_ok=True) # Loading the model and reconstructs the model filename = "outputs/" + args.modelname + ".pt" decoder_state = torch.load(filename) char_rnn = CharRNN(n_characters, 100, n_characters, "gru", 2) char_rnn.load_state_dict(decoder_state) # Generate the predicted sequence based of the hyperparameters predicted = generate(decoder=char_rnn, prime_str=args.prime_str, predict_len=args.predict_len, temperature=args.temperature, cuda=args.cuda) print(predicted)
import os from azureml.core import Workspace, Model from azureml.core.authentication import AzureCliAuthentication ws = Workspace.from_config( auth=AzureCliAuthentication() ) model = Model( workspace=ws, name="net.onnx" ) model.download( target_dir=os.path.join( os.path.dirname(os.path.realpath(__file__)), '../', 'outputs' ) ) print(model)
def load_champion_model(model_name: str) -> Any: champ = Model(workspace, model_name) champ_model_path = champ.download(target_dir='./champion') return joblib.load(champ_model_path)
import azureml.core from azureml.core import Model, Workspace model_name = 'gensim_lda' print("Azure ML SDK Version: ", azureml.core.VERSION) ws = Workspace.from_config() print(ws.name, ws.location, ws.resource_group, sep='\t') model = Model(workspace=ws, name=model_name) model.download()
# download model import json from azureml.core import Workspace, Model, VERSION from azureml.core.authentication import ServicePrincipalAuthentication print(VERSION) with open("aml/config.json", "r") as f: config = json.load(f) auth = ServicePrincipalAuthentication(config["tenant_id"], config["service_principal_id"], config["service_principal_password"]) ws = Workspace.create( name=config["workspace_name"], auth=auth, subscription_id=config['subscription_id'], resource_group=config['resource_group'], exist_ok=True, show_output=True, ) model = Model(ws, name="TinyYOLO") model.download(target_dir="modules/InferenceModule/")