Пример #1
0
    def _check_is_valid_table(table, is_row=False):
        if not isinstance(table, dict):
            azureml_error = AzureMLError.create(
                ArgumentInvalid, argument_name="Table",
                expected_type="dict[string]: column"
            )
            raise AzureMLException._with_error(azureml_error)
        if is_row:
            for key in table:
                val = table[key]
                if isinstance(val, list):
                    azureml_error = AzureMLError.create(
                        InvalidColumnData, type="list", column=key
                    )
                    raise AzureMLException._with_error(azureml_error)
                else:
                    ScalarMetric._check_is_valid_scalar(val)

        keys = list(table.keys())
        if len(keys) > 0:
            reference_column = keys[0]
            table_column_length = TableMetric._get_length(table[reference_column])
            for key in table:
                column_length = TableMetric._get_length(table[key])
                if column_length != table_column_length:
                    azureml_error = AzureMLError.create(
                        InvalidColumnLength, reference_column=reference_column,
                        table_column_length=table_column_length, key=key,
                        column_length=column_length
                    )
                    raise AzureMLException._with_error(azureml_error)
                if isinstance(table[key], list):
                    ListMetric._check_is_valid_list(table[key])
        return table
Пример #2
0
 def create_children(self, tag_key, tag_values, start_children=True):
     """
     Creates one child for each element in tag_values
     :param tag_key: key for the Tags entry to populate in all created children
     :type tag_key: str:
     :param tag_Values: list of values that will map onto Tags[tag_key] for the list of runs created
     :type tag_values: [str]
     :param start_children: Optional flag to start created children, defaults True
     :type start_children: bool:
     :rtype [RunDto]
     """
     request_child_runs = []
     for tag_value in tag_values:
         create_run_dto = CreateRunDto(run_id=RunHistoryFacade.create_run_id(),
                                       parent_run_id=self._run_id,
                                       status='NotStarted',
                                       tags={tag_key: tag_value})
         request_child_runs.append(create_run_dto)
     result_dto = self.run.batch_create_child_runs(request_child_runs)
     errors = result_dto.errors
     if len(errors) > 0:
         azureml_error = AzureMLError.create(
             CreateChildrenFailed, run_id='runid'
         )
         raise AzureMLException._with_error(azureml_error)
     result_child_runs = result_dto.runs
     child_run_ids = [child_run.run_id for child_run in request_child_runs]
     if start_children:
         event_errors = self.run.batch_post_event_start(child_run_ids).errors
         if len(event_errors) > 0:
             azureml_error = AzureMLError.create(
                 StartChildrenFailed, run_id='runid'
             )
             raise AzureMLException._with_error(azureml_error)
     return (result_child_runs[run_id] for run_id in child_run_ids)
Пример #3
0
    def _get_token(self, sdk_resource=None):
        """
        :param sdk_resource: `resource` converted from Track 2 SDK's `scopes`
        """
        external_tenant_tokens = None
        try:
            scheme, token, full_token = self._token_retriever(sdk_resource)
            if self._external_tenant_token_retriever:
                external_tenant_tokens = self._external_tenant_token_retriever(
                    sdk_resource)
        except adal.AdalError as err:
            # pylint: disable=no-member
            if in_cloud_console():
                AdalAuthentication._log_hostname()

            err = (getattr(err, 'error_response', None)
                   or {}).get('error_description') or str(err)
            if 'AADSTS70008' in err:  # all errors starting with 70008 should be creds expiration related
                message = "Please run 'az login'" if not in_cloud_console(
                ) else ''
                azureml_error = AzureMLError.create(
                    CredentialsExpireInactivity, message=message)
                raise AzureMLException._with_error(azureml_error)
            if 'AADSTS50079' in err:
                message = "Please run 'az login'" if not in_cloud_console(
                ) else ''
                azureml_error = AzureMLError.create(
                    AccountConfigurationChanged, message=message)
                raise AzureMLException._with_error(azureml_error)
            if 'AADSTS50173' in err:
                message = "Please clear browser's cookies and run 'az login'" if not in_cloud_console(
                ) else ''
                azureml_error = AzureMLError.create(
                    CredentialExpiredPasswordChanged, message=message)
                raise AzureMLException._with_error(azureml_error)

            raise AzureMLException(err)
        except requests.exceptions.SSLError:
            azureml_error = AzureMLError.create(CertificateVerificationFailure)
            raise AzureMLException._with_error(azureml_error)
        except requests.exceptions.ConnectionError as err:
            azureml_error = AzureMLError.create(NetworkConnectionFailed,
                                                error=str(err))
            raise AzureMLException._with_error(azureml_error)
        except Exception as err:
            if in_cloud_console():
                AdalAuthentication._log_hostname()
            raise err

        return scheme, token, full_token, external_tenant_tokens
Пример #4
0
    def register_submit_function(cls, method_class, submit_function):
        """
        :param cls:
        :type cls: object
        :param method_class:
        :type method_class: str
        :param submit_function:
        :type submit_function: object
        """
        function_name = submit_function.__name__
        module_logger.debug(
            "Trying to register submit_function {}, on method {}".format(
                function_name, method_class))
        with cls._lock:
            if method_class in cls._method_to_submit_dict and \
               cls._method_to_submit_dict[method_class] != submit_function:
                azureml_error = AzureMLError.create(MethodAlreadyRegistered,
                                                    method_class=method_class)
                raise AzureMLException._with_error(azureml_error)

            cls._method_to_submit_dict[method_class] = submit_function

        module_logger.debug(
            "Registered submit_function {}, on method {}".format(
                function_name, method_class))
Пример #5
0
def _write_output_metadata_file(return_object, output_metadata_file_path,
                                logger):
    import errno
    import json
    import os
    logger.debug(
        "Specified output metadata path %s, storing return value type [%s] as json",
        output_metadata_file_path,
        type(return_object).__name__)

    # TODO: Move this to a file utils library in azureml-core
    full_path = os.path.abspath(output_metadata_file_path)
    if os.path.exists(full_path):
        # Can't just use 'x' in open() mode below due to Python 2
        azureml_error = AzureMLError.create(FileAlreadyExists,
                                            full_path=full_path)
        raise AzureMLException._with_error(azureml_error)

    dir_path = os.path.dirname(full_path)
    if not os.path.exists(dir_path):
        try:
            os.makedirs(dir_path)
        except OSError as exc:
            # No Python 3 guarantee for exist_ok :(
            if exc.errno != errno.EEXIST:
                raise

    with open(output_metadata_file_path, 'wt') as omf:
        json.dump(_convert_return_to_json(return_object),
                  omf,
                  indent=4,
                  sort_keys=True)
Пример #6
0
 def log_image(self, name, path=None, plot=None, description=""):
     if path is not None and plot is not None:
         azureml_error = AzureMLError.create(
             TwoInvalidParameter, arg_one="path", arg_two="plot"
         )
         raise AzureMLException._with_error(azureml_error)
     elif path is None and plot is None:
         azureml_error = AzureMLError.create(
             TwoInvalidArgument, arg_one="path", arg_two="plot"
         )
         raise AzureMLException._with_error(azureml_error)
     value = path if path is not None else plot
     metric = ImageMetric(name, value, None, description=description)
     if _use_v2_metrics:
         self._log_metric_v2(metric, is_plot=plot is not None)
     else:
         self._log_metric(metric, is_plot=plot is not None)
Пример #7
0
 def _check_is_valid_scalar(value):
     if not ScalarMetric._is_valid_scalar(value):
         valid_types = list(Metric._type_to_metric_type.keys())
         azureml_error = AzureMLError.create(
             InvalidArgumentType, type=type(value),
             expected_type=valid_types
         )
         raise AzureMLException._with_error(azureml_error)
Пример #8
0
 def _check_is_valid_list(list_value):
     if isinstance(list_value, list):
         for i in range(len(list_value)):
             val = list_value[i]
             if not ScalarMetric._is_valid_scalar(val):
                 valid_types = list(Metric._type_to_metric_type.keys())
                 azureml_error = AzureMLError.create(
                     InvalidArgumentType, type=type(list_value),
                     expected_type=valid_types
                 )
                 raise AzureMLException._with_error(azureml_error)
Пример #9
0
def _process_single_return_object(return_object):
    from msrest.serialization import Model
    if isinstance(return_object, Model):
        object_dict = return_object.as_dict()
        return {_to_camel_case(k): v for k, v in object_dict.items()}
    elif isinstance(return_object, dict):
        return return_object
    else:
        azureml_error = AzureMLError.create(UnsupportedReturnType,
                                            return_object=type(return_object))
        raise AzureMLException._with_error(azureml_error)
Пример #10
0
 def _log_image(self, artifact_client, path, origin, container):
     image_type = imghdr.what(path)
     if image_type is not None:
         artifact_client.upload_artifact(path, origin, container, path,
                                         content_type="image/{}".format(image_type))
     else:
         azureml_error = AzureMLError.create(
             MalformedArgument, argument_name=path
         )
         raise AzureMLException._with_error(azureml_error)
     return path
Пример #11
0
def upload_blob_from_stream(stream,
                            url,
                            content_type=None,
                            session=None,
                            timeout=None,
                            backoff=None,
                            retries=None):
    # TODO add support for upload without azure.storage
    from azureml._vendor.azure_storage.blob import BlockBlobService
    from azureml._vendor.azure_storage.blob.models import ContentSettings
    sas_token, account_name, endpoint_suffix, container_name, blob_name = get_block_blob_service_credentials(
        url)
    content_settings = ContentSettings(content_type=content_type)
    blob_service = BlockBlobService(account_name=account_name,
                                    sas_token=sas_token,
                                    request_session=session,
                                    endpoint_suffix=endpoint_suffix)

    reset_func = StreamResetFunction(stream.tell())

    # Seek to end of stream to validate uploaded blob size matches the local stream size
    stream.seek(0, os.SEEK_END)
    file_size = stream.tell()
    reset_func(stream)

    try:
        from azureml._restclient.clientbase import execute_func_with_reset
        execute_func_with_reset(backoff,
                                retries,
                                blob_service.create_blob_from_stream,
                                reset_func,
                                container_name=container_name,
                                blob_name=blob_name,
                                stream=stream,
                                content_settings=content_settings,
                                timeout=timeout,
                                validate_content=True)
    except AzureHttpError as e:
        if e.status_code == 403:
            azureml_error = AzureMLError.create(
                AuthorizationStorageAccount,
                account_name=account_name,
                container_name=container_name,
                status_code=e.status_code  # , error_code=e.error_code
            )  # error code not present in AzureHttpError
            raise AzureMLException._with_error(azureml_error,
                                               inner_exception=e)
        else:
            raise

    blob_size = blob_service.get_blob_properties(
        container_name, blob_name).properties.content_length
    module_logger.debug("Uploaded blob {} with size {}, file size {}.".format(
        blob_name, blob_size, file_size))
Пример #12
0
    def _is_valid_scalar(value):
        value_type = type(value)
        for number_type in six.integer_types + (float,):
            if isinstance(value, number_type) and sys.getsizeof(value) > AZUREML_MAX_NUMBER_SIZE_IN_BITS:
                azureml_error = AzureMLError.create(
                    ArgumentSizeOutOfRangeType, argument_name=value_type,
                    min=0, max=AZUREML_MAX_NUMBER_SIZE_IN_BITS
                )
                raise AzureMLException._with_error(azureml_error)

        return any(value_type in dictionary
                   for dictionary in (Metric._type_to_metric_type, Metric._type_to_converter))
Пример #13
0
    def flush(self, source, timeout_seconds=None):
        with self._log_context("WaitFlushSource:{}".format(source)) as log_context:

            if timeout_seconds is None:
                log_context.debug("Overriding default flush timeout from None to {}".
                                  format(self._flush_timeout_seconds))
                timeout_seconds = self._flush_timeout_seconds
            else:
                log_context.debug("flush timeout {} is different from task queue timeout {}, using flush timeout".
                                  format(timeout_seconds, self._flush_timeout_seconds))

            start_time = time.time()

            #  Take tasks off of the queue
            tasks_to_wait = []
            while True:
                try:
                    tasks_to_wait.append(self._tasks.get_nowait())
                except Empty:
                    break

            message = ""
            timeout_time = start_time + timeout_seconds

            log_context.debug("Waiting {} seconds on tasks: {}.".format(timeout_seconds, tasks_to_wait))

            not_done = True

            while not_done and time.time() <= timeout_time:
                completed_tasks = [task for task in tasks_to_wait if task.done()]
                tasks_to_wait = [task for task in tasks_to_wait if not task.done()]
                not_done = len(tasks_to_wait) != 0

                self._results.extend((task.wait(awaiter_name=self.identity) for task in completed_tasks))

                if not_done:
                    for task in tasks_to_wait:
                        message += "Waiting on task: {}.\n".format(task.ident)
                    message += "{} tasks left. Current duration of flush {} seconds.\n".format(
                        len(tasks_to_wait), time.time() - start_time)

                    time.sleep(.25)

            self._logger.debug(message)

            # Reach this case on timeout
            if not_done:
                azureml_error = AzureMLError.create(
                    FlushTaskTimeout, timeout_seconds=timeout_seconds
                )
                raise AzureMLException._with_error(azureml_error)
Пример #14
0
    def _log_batch(self, metric_dtos, is_async=False):
        if len(metric_dtos) > AZUREML_MAX_NUMBER_METRICS_BATCH:
            azureml_error = AzureMLError.create(
                MetricsNumberExceeds,
                metric_dtos=len(metric_dtos),
                AZUREML_MAX_NUMBER_METRICS_BATCH=
                AZUREML_MAX_NUMBER_METRICS_BATCH)
            raise AzureMLException._with_error(azureml_error)

        batch_metric_dto = BatchMetricDto(metric_dtos)
        res = self._execute_with_run_arguments(
            self._client.run_metric.post_batch,
            batch_metric_dto,
            is_async=is_async)
        return res
Пример #15
0
 def exec_func():
     metadata_dict = blob_service.get_blob_to_path(
         container_name=container_name,
         blob_name=blob_name,
         file_path=path,
         max_connections=max_concurrency,
         validate_content=_validate_check_sum)
     file_size = os.stat(path).st_size
     module_logger.debug("Downloaded file {} with size {}.".format(
         path, file_size))
     content_length = metadata_dict.properties.content_length
     if (content_length != file_size):
         azureml_error = AzureMLError.create(
             BadDataDownloaded,
             file_size=file_size,
             content_length=content_length)
         raise AzureMLException._with_error(azureml_error)
Пример #16
0
def _retry(exec_func,
           clean_up_func=(lambda: None),
           max_retries=5,
           exceptions=(Exception)):
    """
    A helper function for retry

    :param exec_func: the execution function that runs inside retry mechnism
    :type exec_func: func
    :param clean_up_func: a clean up function that runs inside final statement
    :type clean_up_func: func
    :param max_retries: the number of retries
    :type max_retries: int
    :param exceptions: the exceptions to handle in execution function
    :type stream: Tuple[Type[Exception]]
    :return: results from the return of execution func
    :rtype: AnyType
    """
    wait_time = 2
    retries = 0
    while retries < max_retries:
        try:
            return exec_func()
        except exceptions as request_exception:
            retries += 1
            module_logger.debug(
                'retry has happened in the {} times'.format(retries))
            if retries < max_retries:
                module_logger.debug(
                    'RequestException or HTTPError raised in download_file with message: {}'
                    .format(request_exception))
                time.sleep(wait_time)
                wait_time = wait_time**2
                continue
            else:
                module_logger.error(
                    'Failed to download file with error: {}'.format(
                        request_exception))
                azureml_error = AzureMLError.create(DownloadFailed,
                                                    error=request_exception)
                raise AzureMLException._with_error(azureml_error)
        finally:
            clean_up_func()
Пример #17
0
 def _log_plot(self, artifact_client, plot, origin, container):
     plot_name = self.name + "_" + str(int(time.time()))
     ext = "png"
     artifact_path = "{}.{}".format(plot_name, ext)
     stream = io.BytesIO()
     try:
         plot.savefig(stream, format=ext)
         stream.seek(0)
         artifact_client.upload_artifact(stream, origin, container, artifact_path,
                                         content_type="image/{}".format(ext))
     except AttributeError:
         azureml_error = AzureMLError.create(
             ArgumentInvalid, argument_name="plot",
             expected_type="matplotlib.pyplot"
         )
         raise AzureMLException._with_error(azureml_error)
     finally:
         stream.close()
     return artifact_path
Пример #18
0
    def _log_batch_v2(self, metric_dtos, is_async=False):
        if len(metric_dtos) > AZUREML_MAX_NUMBER_METRICS_BATCH:
            azureml_error = AzureMLError.create(
                MetricsNumberExceeds,
                metric_dtos=len(metric_dtos),
                AZUREML_MAX_NUMBER_METRICS_BATCH=
                AZUREML_MAX_NUMBER_METRICS_BATCH)
            raise AzureMLException._with_error(azureml_error)

        batch_metric_dto = BatchMetricV2Dto(values=metric_dtos,
                                            report_errors=True)
        self._logger.debug(
            "Metrics Client: _log_batch_v2 is calling post_run_metrics "
            "posting {} values.".format(len(batch_metric_dto.values)))
        res = self._execute_with_workspace_run_arguments(
            self._post_run_metrics_log_failed_validations,
            batch_metric_dto,
            is_async=is_async)
        return res
Пример #19
0
 def get_submit_function(cls, method):
     """
     :param cls:
     :type cls: object
     :param method:
     :type method: object
     :return: submit_function
     :rtype: object
     """
     method_class = method.__class__
     module_logger.debug(
         "Trying to get submit_function for method_class {}".format(
             method_class))
     with cls._lock:
         if method_class not in cls._method_to_submit_dict:
             azureml_error = AzureMLError.create(MethodNotRegistered)
             raise AzureMLException._with_error(azureml_error)
         submit_function = cls._method_to_submit_dict[method_class]
         function_name = submit_function.__name__
         module_logger.debug(
             "Retrieved submit_function {} for method {}".format(
                 function_name, method_class))
         return submit_function
Пример #20
0
    def delete_experiment(self, experiment_id, timeout_seconds=600):
        """
        delete empty experiment by experiment_id
        :return: when the delete operation is complete
        """
        call_kwargs = {'raw': True}

        # initial response could be 200 or 202
        initial_response = self._execute_with_workspace_arguments(
            self._client.experiment.delete,
            experiment_id=experiment_id,
            **call_kwargs)

        from .polling import AzureMLPolling
        from msrest.polling.poller import LROPoller

        # "AzureML polling" is a name for the 202/200/location-header contract
        arm_poller = AzureMLPolling(
            timeout=
            5,  # timeout here is actually the delay between polls, bad name
            lro_options={'final-state-via': 'location'})

        # raise an exception to the user when the timeout expires and still got a 202
        def deserialization_callback(response):
            return 1 if response is not None and response.status_code == 202 else 0

        poller = LROPoller(self._client.experiment._client, initial_response,
                           deserialization_callback, arm_poller)

        # this call blocks until the async operation returns 200
        result = poller.result(timeout_seconds)
        if result == 1:
            azureml_error = AzureMLError.create(
                FailedIdWithinSeconds,
                experiment_id=experiment_id,
                timeout_seconds=timeout_seconds)
            raise AzureMLException._with_error(azureml_error)
Пример #21
0
    def get_child_runs(self,
                       root_run_id,
                       recursive=False,
                       _filter_on_server=False,
                       page_size=DEFAULT_PAGE_SIZE,
                       order_by=None,
                       caller=None,
                       custom_headers=None,
                       **kwargs):
        """
        Get child runs by current run_id
        :param root_run_id: optimization id for hierarchy(required)
        :type root_run_id: str
        :param recursive: fetch grandchildren and further descendants(required)
        :type recursive: bool
        :param page_size: number of dto returned by one request (optional)
        :type page_size: int
        :param order_by: keys to sort return values, ('sort_key', 'asc'/'desc')(optional)
        :type order_by: tuple (str, str)
        :param caller: caller function name (optional)
        :type caller: str
        :param custom_headers: headers that will be added to the request (optional)
        :type custom_headers: dict
        :return: list of dictionary whose keys are property of ~_restclient.models.RunDto
        """
        order_by_expression = _validate_order_by(order_by) if order_by else [
            ORDER_BY_STARTTIME_EXPRESSION
        ]
        client_kwargs = _generate_client_kwargs(top=page_size,
                                                orderby=order_by_expression,
                                                caller=caller,
                                                custom_headers=custom_headers,
                                                is_paginated=True)
        client_kwargs.update(kwargs)
        # TODO: _restclient shouldn't depend on core
        if recursive and _filter_on_server and root_run_id != self._run_id:
            azureml_error = AzureMLError.create(
                OnlySupportedServiceSideFiltering)
            raise AzureMLException._with_error(azureml_error)
        elif recursive:
            _filter_on_server = root_run_id == self._run_id or _filter_on_server
            filter_expression = self._get_run_filter_expr(
                **kwargs) if _filter_on_server else None
            root_filter = 'RootRunId eq {0}'.format(root_run_id)
            exclude_parent_filter = 'RunId ne {0}'.format(self._run_id)
            full_filter = and_join([root_filter, filter_expression
                                    ]) if filter_expression else root_filter
            full_filter = and_join([full_filter, exclude_parent_filter
                                    ]) if _filter_on_server else full_filter

            query_params = QueryParamsDto(filter=full_filter)
            run_dtos = self._execute_with_experiment_arguments(
                self._client.run.get_by_query,
                query_params=query_params,
                **client_kwargs)

            if _filter_on_server:
                return run_dtos

            # Filter out nodes outside of the desired sub tree
            run_hierarchy = Tree(run_dtos)
            sub_tree_run_dtos = run_hierarchy.get_subtree_dtos(self._run_id)
            return self._client_filter(sub_tree_run_dtos, **kwargs)

        else:
            run_dtos = self._execute_with_run_arguments(
                self._client.run.get_child, **client_kwargs)
            return run_dtos if _filter_on_server else self._client_filter(
                run_dtos, **kwargs)
Пример #22
0
 def redirect_output_streams_context_manager(self):
     try:
         return self._redirect_output_streams_context_manager
     except AttributeError:
         azureml_error = AzureMLError.create(InvalidOutputStream)
         raise AzureMLException._with_error(azureml_error)