def _check_is_valid_table(table, is_row=False):
if not isinstance(table, dict):
azureml_error = AzureMLError.create(
ArgumentInvalid, argument_name="Table",
expected_type="dict[string]: column"
)
raise AzureMLException._with_error(azureml_error)
if is_row:
for key in table:
val = table[key]
if isinstance(val, list):
azureml_error = AzureMLError.create(
InvalidColumnData, type="list", column=key
)
raise AzureMLException._with_error(azureml_error)
else:
ScalarMetric._check_is_valid_scalar(val)
keys = list(table.keys())
if len(keys) > 0:
reference_column = keys[0]
table_column_length = TableMetric._get_length(table[reference_column])
for key in table:
column_length = TableMetric._get_length(table[key])
if column_length != table_column_length:
azureml_error = AzureMLError.create(
InvalidColumnLength, reference_column=reference_column,
table_column_length=table_column_length, key=key,
column_length=column_length
)
raise AzureMLException._with_error(azureml_error)
if isinstance(table[key], list):
ListMetric._check_is_valid_list(table[key])
return table
def create_children(self, tag_key, tag_values, start_children=True):
"""
Creates one child for each element in tag_values
:param tag_key: key for the Tags entry to populate in all created children
:type tag_key: str:
:param tag_Values: list of values that will map onto Tags[tag_key] for the list of runs created
:type tag_values: [str]
:param start_children: Optional flag to start created children, defaults True
:type start_children: bool:
:rtype [RunDto]
"""
request_child_runs = []
for tag_value in tag_values:
create_run_dto = CreateRunDto(run_id=RunHistoryFacade.create_run_id(),
parent_run_id=self._run_id,
status='NotStarted',
tags={tag_key: tag_value})
request_child_runs.append(create_run_dto)
result_dto = self.run.batch_create_child_runs(request_child_runs)
errors = result_dto.errors
if len(errors) > 0:
azureml_error = AzureMLError.create(
CreateChildrenFailed, run_id='runid'
)
raise AzureMLException._with_error(azureml_error)
result_child_runs = result_dto.runs
child_run_ids = [child_run.run_id for child_run in request_child_runs]
if start_children:
event_errors = self.run.batch_post_event_start(child_run_ids).errors
if len(event_errors) > 0:
azureml_error = AzureMLError.create(
StartChildrenFailed, run_id='runid'
)
raise AzureMLException._with_error(azureml_error)
return (result_child_runs[run_id] for run_id in child_run_ids)
def _get_token(self, sdk_resource=None):
"""
:param sdk_resource: `resource` converted from Track 2 SDK's `scopes`
"""
external_tenant_tokens = None
try:
scheme, token, full_token = self._token_retriever(sdk_resource)
if self._external_tenant_token_retriever:
external_tenant_tokens = self._external_tenant_token_retriever(
sdk_resource)
except adal.AdalError as err:
# pylint: disable=no-member
if in_cloud_console():
AdalAuthentication._log_hostname()
err = (getattr(err, 'error_response', None)
or {}).get('error_description') or str(err)
if 'AADSTS70008' in err: # all errors starting with 70008 should be creds expiration related
message = "Please run 'az login'" if not in_cloud_console(
) else ''
azureml_error = AzureMLError.create(
CredentialsExpireInactivity, message=message)
raise AzureMLException._with_error(azureml_error)
if 'AADSTS50079' in err:
message = "Please run 'az login'" if not in_cloud_console(
) else ''
azureml_error = AzureMLError.create(
AccountConfigurationChanged, message=message)
raise AzureMLException._with_error(azureml_error)
if 'AADSTS50173' in err:
message = "Please clear browser's cookies and run 'az login'" if not in_cloud_console(
) else ''
azureml_error = AzureMLError.create(
CredentialExpiredPasswordChanged, message=message)
raise AzureMLException._with_error(azureml_error)
raise AzureMLException(err)
except requests.exceptions.SSLError:
azureml_error = AzureMLError.create(CertificateVerificationFailure)
raise AzureMLException._with_error(azureml_error)
except requests.exceptions.ConnectionError as err:
azureml_error = AzureMLError.create(NetworkConnectionFailed,
error=str(err))
raise AzureMLException._with_error(azureml_error)
except Exception as err:
if in_cloud_console():
AdalAuthentication._log_hostname()
raise err
return scheme, token, full_token, external_tenant_tokens
def register_submit_function(cls, method_class, submit_function):
"""
:param cls:
:type cls: object
:param method_class:
:type method_class: str
:param submit_function:
:type submit_function: object
"""
function_name = submit_function.__name__
module_logger.debug(
"Trying to register submit_function {}, on method {}".format(
function_name, method_class))
with cls._lock:
if method_class in cls._method_to_submit_dict and \
cls._method_to_submit_dict[method_class] != submit_function:
azureml_error = AzureMLError.create(MethodAlreadyRegistered,
method_class=method_class)
raise AzureMLException._with_error(azureml_error)
cls._method_to_submit_dict[method_class] = submit_function
module_logger.debug(
"Registered submit_function {}, on method {}".format(
function_name, method_class))
def _write_output_metadata_file(return_object, output_metadata_file_path,
logger):
import errno
import json
import os
logger.debug(
"Specified output metadata path %s, storing return value type [%s] as json",
output_metadata_file_path,
type(return_object).__name__)
# TODO: Move this to a file utils library in azureml-core
full_path = os.path.abspath(output_metadata_file_path)
if os.path.exists(full_path):
# Can't just use 'x' in open() mode below due to Python 2
azureml_error = AzureMLError.create(FileAlreadyExists,
full_path=full_path)
raise AzureMLException._with_error(azureml_error)
dir_path = os.path.dirname(full_path)
if not os.path.exists(dir_path):
try:
os.makedirs(dir_path)
except OSError as exc:
# No Python 3 guarantee for exist_ok :(
if exc.errno != errno.EEXIST:
raise
with open(output_metadata_file_path, 'wt') as omf:
json.dump(_convert_return_to_json(return_object),
omf,
indent=4,
sort_keys=True)
def log_image(self, name, path=None, plot=None, description=""):
if path is not None and plot is not None:
azureml_error = AzureMLError.create(
TwoInvalidParameter, arg_one="path", arg_two="plot"
)
raise AzureMLException._with_error(azureml_error)
elif path is None and plot is None:
azureml_error = AzureMLError.create(
TwoInvalidArgument, arg_one="path", arg_two="plot"
)
raise AzureMLException._with_error(azureml_error)
value = path if path is not None else plot
metric = ImageMetric(name, value, None, description=description)
if _use_v2_metrics:
self._log_metric_v2(metric, is_plot=plot is not None)
else:
self._log_metric(metric, is_plot=plot is not None)
def _check_is_valid_scalar(value):
if not ScalarMetric._is_valid_scalar(value):
valid_types = list(Metric._type_to_metric_type.keys())
azureml_error = AzureMLError.create(
InvalidArgumentType, type=type(value),
expected_type=valid_types
)
raise AzureMLException._with_error(azureml_error)
def _check_is_valid_list(list_value):
if isinstance(list_value, list):
for i in range(len(list_value)):
val = list_value[i]
if not ScalarMetric._is_valid_scalar(val):
valid_types = list(Metric._type_to_metric_type.keys())
azureml_error = AzureMLError.create(
InvalidArgumentType, type=type(list_value),
expected_type=valid_types
)
raise AzureMLException._with_error(azureml_error)
def _process_single_return_object(return_object):
from msrest.serialization import Model
if isinstance(return_object, Model):
object_dict = return_object.as_dict()
return {_to_camel_case(k): v for k, v in object_dict.items()}
elif isinstance(return_object, dict):
return return_object
else:
azureml_error = AzureMLError.create(UnsupportedReturnType,
return_object=type(return_object))
raise AzureMLException._with_error(azureml_error)
def _log_image(self, artifact_client, path, origin, container):
image_type = imghdr.what(path)
if image_type is not None:
artifact_client.upload_artifact(path, origin, container, path,
content_type="image/{}".format(image_type))
else:
azureml_error = AzureMLError.create(
MalformedArgument, argument_name=path
)
raise AzureMLException._with_error(azureml_error)
return path
def upload_blob_from_stream(stream,
url,
content_type=None,
session=None,
timeout=None,
backoff=None,
retries=None):
# TODO add support for upload without azure.storage
from azureml._vendor.azure_storage.blob import BlockBlobService
from azureml._vendor.azure_storage.blob.models import ContentSettings
sas_token, account_name, endpoint_suffix, container_name, blob_name = get_block_blob_service_credentials(
url)
content_settings = ContentSettings(content_type=content_type)
blob_service = BlockBlobService(account_name=account_name,
sas_token=sas_token,
request_session=session,
endpoint_suffix=endpoint_suffix)
reset_func = StreamResetFunction(stream.tell())
# Seek to end of stream to validate uploaded blob size matches the local stream size
stream.seek(0, os.SEEK_END)
file_size = stream.tell()
reset_func(stream)
try:
from azureml._restclient.clientbase import execute_func_with_reset
execute_func_with_reset(backoff,
retries,
blob_service.create_blob_from_stream,
reset_func,
container_name=container_name,
blob_name=blob_name,
stream=stream,
content_settings=content_settings,
timeout=timeout,
validate_content=True)
except AzureHttpError as e:
if e.status_code == 403:
azureml_error = AzureMLError.create(
AuthorizationStorageAccount,
account_name=account_name,
container_name=container_name,
status_code=e.status_code # , error_code=e.error_code
) # error code not present in AzureHttpError
raise AzureMLException._with_error(azureml_error,
inner_exception=e)
else:
raise
blob_size = blob_service.get_blob_properties(
container_name, blob_name).properties.content_length
module_logger.debug("Uploaded blob {} with size {}, file size {}.".format(
blob_name, blob_size, file_size))
def _is_valid_scalar(value):
value_type = type(value)
for number_type in six.integer_types + (float,):
if isinstance(value, number_type) and sys.getsizeof(value) > AZUREML_MAX_NUMBER_SIZE_IN_BITS:
azureml_error = AzureMLError.create(
ArgumentSizeOutOfRangeType, argument_name=value_type,
min=0, max=AZUREML_MAX_NUMBER_SIZE_IN_BITS
)
raise AzureMLException._with_error(azureml_error)
return any(value_type in dictionary
for dictionary in (Metric._type_to_metric_type, Metric._type_to_converter))
def flush(self, source, timeout_seconds=None):
with self._log_context("WaitFlushSource:{}".format(source)) as log_context:
if timeout_seconds is None:
log_context.debug("Overriding default flush timeout from None to {}".
format(self._flush_timeout_seconds))
timeout_seconds = self._flush_timeout_seconds
else:
log_context.debug("flush timeout {} is different from task queue timeout {}, using flush timeout".
format(timeout_seconds, self._flush_timeout_seconds))
start_time = time.time()
# Take tasks off of the queue
tasks_to_wait = []
while True:
try:
tasks_to_wait.append(self._tasks.get_nowait())
except Empty:
break
message = ""
timeout_time = start_time + timeout_seconds
log_context.debug("Waiting {} seconds on tasks: {}.".format(timeout_seconds, tasks_to_wait))
not_done = True
while not_done and time.time() <= timeout_time:
completed_tasks = [task for task in tasks_to_wait if task.done()]
tasks_to_wait = [task for task in tasks_to_wait if not task.done()]
not_done = len(tasks_to_wait) != 0
self._results.extend((task.wait(awaiter_name=self.identity) for task in completed_tasks))
if not_done:
for task in tasks_to_wait:
message += "Waiting on task: {}.\n".format(task.ident)
message += "{} tasks left. Current duration of flush {} seconds.\n".format(
len(tasks_to_wait), time.time() - start_time)
time.sleep(.25)
self._logger.debug(message)
# Reach this case on timeout
if not_done:
azureml_error = AzureMLError.create(
FlushTaskTimeout, timeout_seconds=timeout_seconds
)
raise AzureMLException._with_error(azureml_error)
def _log_batch(self, metric_dtos, is_async=False):
if len(metric_dtos) > AZUREML_MAX_NUMBER_METRICS_BATCH:
azureml_error = AzureMLError.create(
MetricsNumberExceeds,
metric_dtos=len(metric_dtos),
AZUREML_MAX_NUMBER_METRICS_BATCH=
AZUREML_MAX_NUMBER_METRICS_BATCH)
raise AzureMLException._with_error(azureml_error)
batch_metric_dto = BatchMetricDto(metric_dtos)
res = self._execute_with_run_arguments(
self._client.run_metric.post_batch,
batch_metric_dto,
is_async=is_async)
return res
def exec_func():
metadata_dict = blob_service.get_blob_to_path(
container_name=container_name,
blob_name=blob_name,
file_path=path,
max_connections=max_concurrency,
validate_content=_validate_check_sum)
file_size = os.stat(path).st_size
module_logger.debug("Downloaded file {} with size {}.".format(
path, file_size))
content_length = metadata_dict.properties.content_length
if (content_length != file_size):
azureml_error = AzureMLError.create(
BadDataDownloaded,
file_size=file_size,
content_length=content_length)
raise AzureMLException._with_error(azureml_error)
def _retry(exec_func,
clean_up_func=(lambda: None),
max_retries=5,
exceptions=(Exception)):
"""
A helper function for retry
:param exec_func: the execution function that runs inside retry mechnism
:type exec_func: func
:param clean_up_func: a clean up function that runs inside final statement
:type clean_up_func: func
:param max_retries: the number of retries
:type max_retries: int
:param exceptions: the exceptions to handle in execution function
:type stream: Tuple[Type[Exception]]
:return: results from the return of execution func
:rtype: AnyType
"""
wait_time = 2
retries = 0
while retries < max_retries:
try:
return exec_func()
except exceptions as request_exception:
retries += 1
module_logger.debug(
'retry has happened in the {} times'.format(retries))
if retries < max_retries:
module_logger.debug(
'RequestException or HTTPError raised in download_file with message: {}'
.format(request_exception))
time.sleep(wait_time)
wait_time = wait_time**2
continue
else:
module_logger.error(
'Failed to download file with error: {}'.format(
request_exception))
azureml_error = AzureMLError.create(DownloadFailed,
error=request_exception)
raise AzureMLException._with_error(azureml_error)
finally:
clean_up_func()
def _log_plot(self, artifact_client, plot, origin, container):
plot_name = self.name + "_" + str(int(time.time()))
ext = "png"
artifact_path = "{}.{}".format(plot_name, ext)
stream = io.BytesIO()
try:
plot.savefig(stream, format=ext)
stream.seek(0)
artifact_client.upload_artifact(stream, origin, container, artifact_path,
content_type="image/{}".format(ext))
except AttributeError:
azureml_error = AzureMLError.create(
ArgumentInvalid, argument_name="plot",
expected_type="matplotlib.pyplot"
)
raise AzureMLException._with_error(azureml_error)
finally:
stream.close()
return artifact_path
def _log_batch_v2(self, metric_dtos, is_async=False):
if len(metric_dtos) > AZUREML_MAX_NUMBER_METRICS_BATCH:
azureml_error = AzureMLError.create(
MetricsNumberExceeds,
metric_dtos=len(metric_dtos),
AZUREML_MAX_NUMBER_METRICS_BATCH=
AZUREML_MAX_NUMBER_METRICS_BATCH)
raise AzureMLException._with_error(azureml_error)
batch_metric_dto = BatchMetricV2Dto(values=metric_dtos,
report_errors=True)
self._logger.debug(
"Metrics Client: _log_batch_v2 is calling post_run_metrics "
"posting {} values.".format(len(batch_metric_dto.values)))
res = self._execute_with_workspace_run_arguments(
self._post_run_metrics_log_failed_validations,
batch_metric_dto,
is_async=is_async)
return res
def get_submit_function(cls, method):
"""
:param cls:
:type cls: object
:param method:
:type method: object
:return: submit_function
:rtype: object
"""
method_class = method.__class__
module_logger.debug(
"Trying to get submit_function for method_class {}".format(
method_class))
with cls._lock:
if method_class not in cls._method_to_submit_dict:
azureml_error = AzureMLError.create(MethodNotRegistered)
raise AzureMLException._with_error(azureml_error)
submit_function = cls._method_to_submit_dict[method_class]
function_name = submit_function.__name__
module_logger.debug(
"Retrieved submit_function {} for method {}".format(
function_name, method_class))
return submit_function
def delete_experiment(self, experiment_id, timeout_seconds=600):
"""
delete empty experiment by experiment_id
:return: when the delete operation is complete
"""
call_kwargs = {'raw': True}
# initial response could be 200 or 202
initial_response = self._execute_with_workspace_arguments(
self._client.experiment.delete,
experiment_id=experiment_id,
**call_kwargs)
from .polling import AzureMLPolling
from msrest.polling.poller import LROPoller
# "AzureML polling" is a name for the 202/200/location-header contract
arm_poller = AzureMLPolling(
timeout=
5, # timeout here is actually the delay between polls, bad name
lro_options={'final-state-via': 'location'})
# raise an exception to the user when the timeout expires and still got a 202
def deserialization_callback(response):
return 1 if response is not None and response.status_code == 202 else 0
poller = LROPoller(self._client.experiment._client, initial_response,
deserialization_callback, arm_poller)
# this call blocks until the async operation returns 200
result = poller.result(timeout_seconds)
if result == 1:
azureml_error = AzureMLError.create(
FailedIdWithinSeconds,
experiment_id=experiment_id,
timeout_seconds=timeout_seconds)
raise AzureMLException._with_error(azureml_error)
def get_child_runs(self,
root_run_id,
recursive=False,
_filter_on_server=False,
page_size=DEFAULT_PAGE_SIZE,
order_by=None,
caller=None,
custom_headers=None,
**kwargs):
"""
Get child runs by current run_id
:param root_run_id: optimization id for hierarchy(required)
:type root_run_id: str
:param recursive: fetch grandchildren and further descendants(required)
:type recursive: bool
:param page_size: number of dto returned by one request (optional)
:type page_size: int
:param order_by: keys to sort return values, ('sort_key', 'asc'/'desc')(optional)
:type order_by: tuple (str, str)
:param caller: caller function name (optional)
:type caller: str
:param custom_headers: headers that will be added to the request (optional)
:type custom_headers: dict
:return: list of dictionary whose keys are property of ~_restclient.models.RunDto
"""
order_by_expression = _validate_order_by(order_by) if order_by else [
ORDER_BY_STARTTIME_EXPRESSION
]
client_kwargs = _generate_client_kwargs(top=page_size,
orderby=order_by_expression,
caller=caller,
custom_headers=custom_headers,
is_paginated=True)
client_kwargs.update(kwargs)
# TODO: _restclient shouldn't depend on core
if recursive and _filter_on_server and root_run_id != self._run_id:
azureml_error = AzureMLError.create(
OnlySupportedServiceSideFiltering)
raise AzureMLException._with_error(azureml_error)
elif recursive:
_filter_on_server = root_run_id == self._run_id or _filter_on_server
filter_expression = self._get_run_filter_expr(
**kwargs) if _filter_on_server else None
root_filter = 'RootRunId eq {0}'.format(root_run_id)
exclude_parent_filter = 'RunId ne {0}'.format(self._run_id)
full_filter = and_join([root_filter, filter_expression
]) if filter_expression else root_filter
full_filter = and_join([full_filter, exclude_parent_filter
]) if _filter_on_server else full_filter
query_params = QueryParamsDto(filter=full_filter)
run_dtos = self._execute_with_experiment_arguments(
self._client.run.get_by_query,
query_params=query_params,
**client_kwargs)
if _filter_on_server:
return run_dtos
# Filter out nodes outside of the desired sub tree
run_hierarchy = Tree(run_dtos)
sub_tree_run_dtos = run_hierarchy.get_subtree_dtos(self._run_id)
return self._client_filter(sub_tree_run_dtos, **kwargs)
else:
run_dtos = self._execute_with_run_arguments(
self._client.run.get_child, **client_kwargs)
return run_dtos if _filter_on_server else self._client_filter(
run_dtos, **kwargs)
def redirect_output_streams_context_manager(self):
try:
return self._redirect_output_streams_context_manager
except AttributeError:
azureml_error = AzureMLError.create(InvalidOutputStream)
raise AzureMLException._with_error(azureml_error)