def get_or_create_workspace_from_project(
project_configuration: ProjectConfiguration,
auth: Union[
InteractiveLoginAuthentication,
ServicePrincipalAuthentication] = InteractiveLoginAuthentication(),
show_output=True,
) -> Workspace:
"""
Create a new Azure Machine Learning workspace. If the workspace already exists, the existing workspace will be
returned. Also create a CONFIG file to quickly reload the workspace.
This uses the :class:`azureml.core.authentication.InteractiveLoginAuthentication` or will default to use the
:class:`azureml.core.authentication.AzureCliAuthentication` for logging into Azure.
Run az login from the CLI in the project directory to avoid authentication when running the program.
:param project_configuration: Project Configuration Container
:param auth: Derived classes provide different means to authenticate and acquire a token based on their targeted
use case.
For examples of authentication, see https://aka.ms/aml-notebook-auth.
:type auth: azureml.core.authentication.AbstractAuthentication
:param show_output: enable print output
:return: Returns a :class:`azureml.core.Workspace` object, a pointer to Azure Machine Learning Workspace
Learning Workspace
"""
return get_or_create_workspace(
project_configuration.get_value("workspace_name"),
project_configuration.get_value("subscription_id"),
project_configuration.get_value("resource_group"),
project_configuration.get_value("workspace_region"),
auth=auth,
log=show_output,
)
def __init__(self,
subscription_id: str,
resource_group: str,
workspace_name: str,
configuration_file: str = project_configuration_file,
project_configuration=None,
train_py: str = train_py_default,
score_py: str = score_py_default,
**kwargs):
"""
Interface Constructor for Workspace Context
:param subscription_id: Azure subscription id
:param resource_group: Azure Resource Group name
:param workspace_name: Azure Machine Learning Workspace
:param configuration_file: path to project configuration file. default: project.yml
:param train_py: python source file for training
:param score_py: python source file for scoring
"""
super().__init__(subscription_id, resource_group, workspace_name,
**kwargs)
if not project_configuration:
self.project_configuration = ProjectConfiguration(
configuration_file)
self.image_tags = None
self.args = None
self.train_py = train_py
self.score_py = score_py
self.show_output = True
self.source_directory = "./script"
self.experiment_name = None
self.model_name = None
self.wait_for_completion = True
self.model_path = None
def update_and_save_configuration(proj_config: ProjectConfiguration,
setting_boxes: dict, name2id: dict):
"""
Update project configuration with widget values and save
:param proj_config: Project Configuration loaded from file.
:param setting_boxes: Dict of Each Settings Widget
:param name2id: Mapping of Subscription Name to ID
"""
for boxes in setting_boxes:
if boxes == "subscription_id":
proj_config.set_value(boxes, name2id[setting_boxes[boxes].value])
else:
proj_config.set_value(boxes, setting_boxes[boxes].value)
save_project_configuration(proj_config)
def configure_settings(configuration_yaml: str = project_configuration_file):
"""
Launch a tkinter UI to configure the project settings in the provided
configuration_yaml file. If a file is not provided, the default ./project.yml
file will be created for the caller.
configuration_yaml -> Disk location of the configuration file to modify.
ProjectConfiguration will open an existing YAML file or create a new one. It is
suggested that your project simply create a simple configuration file containing
all of you settings so that the user simply need to modify it with the UI.
In this instance, we assume that the default configuration file is called project.yml.
This will be used if the user passes nothing else in.
:param configuration_yaml: Location of configuration yaml
"""
project_configuration = ProjectConfiguration(configuration_yaml)
# Finally, create a Tk window and pass that along with the configuration object
# to the SettingsObject class for modification.
window = Tk()
app = SettingsUpdate(project_configuration, window)
app.mainloop()
def get_or_create_image(
image_config,
image_settings_name,
show_output,
models=None,
configuration_file: str = project_configuration_file,
):
"""
:param image_config:
:param image_settings_name:
:param models:
:param show_output:
:param configuration_file: path to project configuration file. default: project.yml
:return:
"""
if not models:
models = []
project_configuration = ProjectConfiguration(configuration_file)
assert project_configuration.has_value(image_settings_name)
image_name = project_configuration.get_value(image_settings_name)
workspace = get_or_create_workspace_from_project(
project_configuration, show_output=show_output
)
workspace_images = workspace.images
if (
image_name in workspace_images
and workspace_images[image_name].creation_state != "Failed"
):
return workspace_images[image_name]
image_create_start = time.time()
image = ContainerImage.create(
name=image_name, models=models, image_config=image_config, workspace=workspace
)
image.wait_for_creation(show_output=show_output)
assert image.creation_state != "Failed"
if show_output:
print_image_deployment_info(image, image_name, image_create_start)
return image
def update_setting_boxes(new_proj_config: ProjectConfiguration,
setting_boxes: dict, id2name: dict):
"""
Update Settings with new Project Configuration
:param new_proj_config: Newly loaded project configuration
:param setting_boxes: list of setting widgets
:param id2name: mapping of sub ids to sub names
"""
for setting_box_key in setting_boxes:
if new_proj_config.has_value(setting_box_key):
if setting_box_key == "subscription_id":
setting_boxes[setting_box_key].value = id2name[
new_proj_config.get_value(setting_box_key)]
else:
setting_boxes[
setting_box_key].value = new_proj_config.get_value(
setting_box_key)
else:
warnings.warn("Reload Widget to display new properties")
def unique_configuration(self):
project_configuration = ProjectConfiguration(
project_configuration_file)
allchar = string.ascii_letters + string.digits
append = "".join(random.choice(allchar) for _ in range(1, 5))
settings = [
"resource_group",
"workspace_name",
"image_name",
"aks_service_name",
"aks_location",
"aks_name",
"deep_image_name",
"deep_aks_service_name",
"deep_aks_name",
"deep_aks_location",
]
for setting in settings:
project_configuration.append_value(setting, append)
yield project_configuration
ws = WorkspaceContext.get_or_create_workspace(
project_configuration=project_configuration)
rg_client = ResourceManagementClient(
ws._auth, project_configuration.get_value("subscription_id"))
rg_client.resource_groups.delete(
resource_group_name=project_configuration.get_value(
"resource_group"))
def create_settings_boxes(default_sub: str, name2id: dict,
proj_config: ProjectConfiguration) -> dict:
"""
Create Settings Boxes
:param default_sub: Sub to set as current selection
:param name2id: sub name to id mapping
:param proj_config: Project Configuration Container
:return: Text Widgets for each setting.
"""
setting_boxs = {}
user_id = getpass.getuser()
for setting in proj_config.get_settings():
for setting_key in setting:
setting_with_id = proj_config.get_value(setting_key).replace(
"$(User)", user_id)
proj_config.set_value(setting_key, setting_with_id)
setting = setting[setting_key][0]
description = setting["description"]
if setting_key == "subscription_id":
setting_boxs["subscription_id"] = widgets.Dropdown(
options=list(name2id.keys()) + [""],
value=default_sub.replace("<>", ""),
description="subscription_id",
disabled=False,
style={"description_width": "initial"},
layout=Layout(width="50%"),
)
else:
setting_boxs[setting_key] = widgets.Text(
value=setting_with_id.replace("<>", ""),
placeholder=description,
description=setting_key,
disabled=False,
style={"description_width": "initial"},
layout=Layout(width="50%"),
)
return setting_boxs
def upload_on_change(change: dict):
"""
:param change:
"""
if change["type"] == "change" and change["name"] == "value":
for file in uploader.value:
with open(config, "wb") as f:
f.write(uploader.value[file]["content"])
f.close()
new_proj_config = ProjectConfiguration(config)
update_setting_boxes(new_proj_config, setting_boxes, id2name)
def get_or_create_workspace(
cls,
configuration_file: str = project_configuration_file,
**kwargs):
project_configuration = ProjectConfiguration(configuration_file)
assert project_configuration.has_value("subscription_id")
assert project_configuration.has_value("resource_group")
assert project_configuration.has_value("workspace_name")
assert project_configuration.has_value("workspace_region")
return MockMLRealtimeScore(
project_configuration.get_value("subscription_id"),
project_configuration.get_value("resource_group"),
project_configuration.get_value("workspace_name"),
project_configuration_file,
**kwargs,
)
def mockreturn(train_py, score_py):
project_configuration = ProjectConfiguration(
project_configuration_file)
assert project_configuration.has_value("subscription_id")
assert project_configuration.has_value("resource_group")
assert project_configuration.has_value("workspace_name")
ws = MockMLRealtimeScore(
subscription_id=project_configuration.get_value(
"subscription_id"),
resource_group=project_configuration.get_value(
"resource_group"),
workspace_name=project_configuration.get_value(
"workspace_name"),
configuration_file=project_configuration_file,
score_py=score_py,
train_py=train_py,
)
return ws
def get_settings(
configuration_yaml: str = project_configuration_file,
) -> ProjectConfiguration:
"""
Acquire the project settings from the provided configuration_yaml file.
If a file is not provided, the default ./project.yml will be created and
and empty set of settings will be returned to the user.
configuration_yaml -> Disk location of the configuration file to modify.
ProjectConfiguration will open an existing YAML file or create a new one. It is
suggested that your project simply create a simple configuration file containing
all of you settings so that the user simply need to modify it with the UI.
In this instance, we assume that the default configuration file is called project.yml.
This will be used if the user passes nothing else in.
:param configuration_yaml: Project configuration yml
:return: loaded ProjectConfiguration object
"""
return ProjectConfiguration(configuration_yaml)
def get_configuration_widget(config: str, with_existing: bool = True) -> VBox:
"""
Get Configuration Widget for Configuration File
:param config: project configuration filename
:param with_existing:
:return:
"""
proj_config = ProjectConfiguration(config)
proj_config.save_configuration()
out = widgets.Output()
uploader = widgets.FileUpload(accept=".yml",
multiple=False,
layout=Layout(width="50%"))
name2id, id2name = list_subscriptions()
# name2id = {'sub_id': proj_config.get_value("subscription_id")}
# id2name = {proj_config.get_value("subscription_id"): "sub_id"}
getpass.getuser()
# default_sub = proj_config.get_value("subscription_id")
default_sub = list(name2id.keys())[0]
if proj_config.get_value("subscription_id") in id2name:
default_sub = id2name[proj_config.get_value("subscription_id")]
else:
proj_config.set_value("subscription_id", list(id2name.keys())[0])
proj_config.save_configuration()
setting_boxes = create_settings_boxes(default_sub, name2id, proj_config)
proj_config.save_configuration()
def convert_to_region(key: str) -> None:
"""
:param key:
"""
if key in setting_boxes:
setting_boxes[key] = widgets.Dropdown(
options=[
"eastus",
"eastus2",
"canadacentral",
"centralus",
"northcentralus",
"southcentralus",
"westcentralus",
"westus",
"westus2",
],
value=proj_config.get_value(key).replace("<>", "eastus"),
description=key,
disabled=False,
style={"description_width": "initial"},
layout=Layout(width="50%"),
)
convert_to_region("workspace_region")
convert_to_region("aks_location")
convert_to_region("deep_aks_location")
dropdown_keys = []
my_list = get_widgets_list(dropdown_keys, out, setting_boxes, uploader,
with_existing, config)
def upload_on_change(change: dict):
"""
:param change:
"""
if change["type"] == "change" and change["name"] == "value":
for file in uploader.value:
with open(config, "wb") as f:
f.write(uploader.value[file]["content"])
f.close()
new_proj_config = ProjectConfiguration(config)
update_setting_boxes(new_proj_config, setting_boxes, id2name)
uploader.observe(upload_on_change)
def on_change(change: dict):
"""
:param change:
"""
if change["type"] == "change" and change["name"] == "value":
update_and_save_configuration(proj_config, setting_boxes, name2id)
for box in setting_boxes:
setting_boxes[box].observe(on_change)
return widgets.VBox(my_list)
def get_or_create_workspace(
cls,
configuration_file: str = project_configuration_file,
project_configuration: ProjectConfiguration = None,
**kwargs):
""" Get or create a workspace if it doesn't exist.
:param configuration_file:
:param project_configuration: ProjectConfiguration
"""
if not project_configuration:
project_configuration = ProjectConfiguration(configuration_file)
assert project_configuration.has_value("subscription_id")
assert project_configuration.has_value("resource_group")
assert project_configuration.has_value("workspace_name")
assert project_configuration.has_value("workspace_region")
try:
check_valid_resource_name(
project_configuration.get_value("workspace_name"), "Workspace")
except UserErrorException:
print(project_configuration.get_value("workspace_name"))
raise
cls.create(
subscription_id=project_configuration.get_value("subscription_id"),
resource_group=project_configuration.get_value("resource_group"),
name=project_configuration.get_value("workspace_name"),
location=project_configuration.get_value("workspace_region"),
exist_ok=True,
)
ws = cls(
subscription_id=project_configuration.get_value("subscription_id"),
resource_group=project_configuration.get_value("resource_group"),
workspace_name=project_configuration.get_value("workspace_name"),
project_configuration=project_configuration,
**kwargs)
return ws
class WorkspaceContext(Workspace):
"""
AzureML Workspace Context - Base Framework Interface
"""
def __init__(self,
subscription_id: str,
resource_group: str,
workspace_name: str,
configuration_file: str = project_configuration_file,
project_configuration=None,
train_py: str = train_py_default,
score_py: str = score_py_default,
**kwargs):
"""
Interface Constructor for Workspace Context
:param subscription_id: Azure subscription id
:param resource_group: Azure Resource Group name
:param workspace_name: Azure Machine Learning Workspace
:param configuration_file: path to project configuration file. default: project.yml
:param train_py: python source file for training
:param score_py: python source file for scoring
"""
super().__init__(subscription_id, resource_group, workspace_name,
**kwargs)
if not project_configuration:
self.project_configuration = ProjectConfiguration(
configuration_file)
self.image_tags = None
self.args = None
self.train_py = train_py
self.score_py = score_py
self.show_output = True
self.source_directory = "./script"
self.experiment_name = None
self.model_name = None
self.wait_for_completion = True
self.model_path = None
@classmethod
def get_or_create_workspace(
cls,
configuration_file: str = project_configuration_file,
project_configuration: ProjectConfiguration = None,
**kwargs):
""" Get or create a workspace if it doesn't exist.
:param configuration_file:
:param project_configuration: ProjectConfiguration
"""
if not project_configuration:
project_configuration = ProjectConfiguration(configuration_file)
assert project_configuration.has_value("subscription_id")
assert project_configuration.has_value("resource_group")
assert project_configuration.has_value("workspace_name")
assert project_configuration.has_value("workspace_region")
try:
check_valid_resource_name(
project_configuration.get_value("workspace_name"), "Workspace")
except UserErrorException:
print(project_configuration.get_value("workspace_name"))
raise
cls.create(
subscription_id=project_configuration.get_value("subscription_id"),
resource_group=project_configuration.get_value("resource_group"),
name=project_configuration.get_value("workspace_name"),
location=project_configuration.get_value("workspace_region"),
exist_ok=True,
)
ws = cls(
subscription_id=project_configuration.get_value("subscription_id"),
resource_group=project_configuration.get_value("resource_group"),
workspace_name=project_configuration.get_value("workspace_name"),
project_configuration=project_configuration,
**kwargs)
return ws
@staticmethod
def _get_file_md5(file_name: str) -> str:
hasher = hashlib.md5()
with open(file_name, "rb") as afile:
buf = afile.read()
hasher.update(buf)
file_hash = hasher.hexdigest()
return file_hash
def assert_and_get_value(self, setting_name: str) -> str:
"""
:param setting_name:
:return:
"""
assert self.project_configuration.has_value(setting_name)
return self.project_configuration.get_value(setting_name)
def dont_test_config() -> None:
new_config_file = "./testconfiguration.yml"
project_name = "Test Project"
# Make sure file doesn't exist
remove_config_file(new_config_file)
# Create a new one with a specific name and two settings
proj_config = ProjectConfiguration(new_config_file)
proj_config.set_project_name(project_name)
proj_config.add_setting("sub_id", "Your Azure Subscription", "my_sub")
proj_config.add_setting("workspace", "Your Azure ML Workspace", "my_ws")
assert proj_config.project_name() == project_name
assert proj_config.get_value('sub_id') == 'my_sub'
assert proj_config.get_value('workspace') == 'my_ws'
assert len(proj_config.get_settings()) == 14
# Save it and ensure the file exists
assert proj_config.configuration
proj_config.save_configuration()
assert os.path.isfile(new_config_file)
# Load it and check what we have
proj_config = ProjectConfiguration(new_config_file)
assert proj_config.project_name() == project_name
assert proj_config.get_value('sub_id') == 'my_sub'
assert proj_config.get_value('workspace') == 'my_ws'
assert len(proj_config.get_settings()) == 14
# Change a setting and test we get the right value
proj_config.set_value('sub_id', 'new_sub')
assert proj_config.get_value('sub_id') == 'new_sub'
remove_config_file(new_config_file)
file1 = "this_is_file.txt"
open(file1, "w+")
found, path = find_file(file1)
assert found
os.remove(file1)
found, path = find_file("not_this_is_file.txt")
assert not found
file2 = "../this_is_higher_file.txt"
open(file2, "w+")
found, path = find_file(file2)
assert found
os.remove(file2)