def as_dataframe(self):
"""
Returns this collection of Experiment Runs as a table.
Returns
-------
:class:`pandas.DataFrame`
"""
pd = importer.maybe_dependency("pandas")
if pd is None:
e = ImportError("pandas is not installed; try `pip install pandas`")
six.raise_from(e, None)
ids = []
data = []
for run in self:
run_data = {}
run_data.update({'hpp.'+k: v for k, v in run.get_hyperparameters().items()})
run_data.update({'metric.'+k: v for k, v in run.get_metrics().items()})
ids.append(run.id)
data.append(run_data)
columns = set()
for run_data in data:
columns.update(run_data.keys())
return pd.DataFrame(data, index=ids, columns=sorted(list(columns)))
def _from_dict(cls, d):
# NOTE: ignores extraneous keys in `d`
try:
input_topic = d["input_topic"]
output_topic = d["output_topic"]
error_topic = d["error_topic"]
except KeyError as e:
msg = 'expected but did not find key "{}"'.format(e.args[0])
six.raise_from(RuntimeError(msg), None)
return cls(input_topic, output_topic, error_topic)
def _capture_env_vars(self, env_vars):
if env_vars is None:
return
try:
env_vars_dict = {name: os.environ[name] for name in env_vars}
except KeyError as e:
new_e = KeyError("'{}' not found in environment".format(e.args[0]))
six.raise_from(new_e, None)
self._msg.environment_variables.extend(
_EnvironmentService.EnvironmentVariablesBlob(name=name,
value=value)
for name, value in six.viewitems(env_vars_dict))
def download_docker_context(self, download_to_path, self_contained=False):
"""
Downloads this Model Version's Docker context ``tgz``.
Parameters
----------
download_to_path : str
Path to download Docker context to.
self_contained : bool, default False
Whether the downloaded Docker context should be self-contained.
Returns
-------
downloaded_to_path : str
Absolute path where Docker context was downloaded to. Matches `download_to_path`.
"""
self._refresh_cache()
endpoint = "{}://{}/api/v1/deployment/builds/dockercontext".format(
self._conn.scheme,
self._conn.socket,
)
body = {
"model_version_id": self.id,
"self_contained": self_contained,
}
with _utils.make_request("POST",
endpoint,
self._conn,
json=body,
stream=True) as response:
try:
_utils.raise_for_http_error(response)
except requests.HTTPError as e:
# propagate error caused by missing artifact
error_text = e.response.text.strip()
if error_text.startswith("missing artifact"):
new_e = RuntimeError(
"unable to obtain Docker context due to " + error_text)
six.raise_from(new_e, None)
else:
raise e
downloaded_to_path = _request_utils.download_file(
response, download_to_path, overwrite_ok=True)
return os.path.abspath(downloaded_to_path)
def _check_compatible_value_helper(value, value_name, allowed_types):
try:
json.dumps(value)
except TypeError as e:
err_msg = "{}; {} must only contain types {}".format(
str(e),
value_name,
sorted(map(lambda cls: cls.__name__, allowed_types)),
)
six.raise_from(TypeError(err_msg), None)
except UnicodeDecodeError as e:
# in Python 2, json.dumps() attempts to decode binary (unlike Python 3
# which rejects it outright), so here we clarify the potential error
err_msg = "{}; {} cannot contain binary; consider encoding to base64 instead".format(
str(e),
value_name,
)
six.raise_from(TypeError(err_msg), None)
def log_setup_script(self, script, overwrite=False):
"""
Associate a model deployment setup script with this Experiment Run.
.. versionadded:: 0.13.8
Parameters
----------
script : str
String composed of valid Python code for executing setup steps at the beginning of model
deployment. An on-disk file can be passed in using ``open("path/to/file.py", 'r').read()``.
overwrite : bool, default False
Whether to allow overwriting an existing setup script.
Raises
------
SyntaxError
If `script` contains invalid Python.
"""
# validate `script`'s syntax
try:
ast.parse(script)
except SyntaxError as e:
# clarify that the syntax error comes from `script`, and propagate details
reason = e.args[0]
line_no = e.args[1][1]
line = script.splitlines()[line_no - 1]
six.raise_from(
SyntaxError("{} in provided script on line {}:\n{}".format(
reason, line_no, line)), e)
# convert into bytes for upload
script = six.ensure_binary(script)
# convert to file-like for `_log_artifact()`
script = six.BytesIO(script)
self.log_artifact(
"setup_script",
script,
overwrite,
"py",
)
def add_env_vars(self, env_vars):
"""Add environment variables.
Parameters
----------
env_vars : list of str, or dict of str to str
Environment variables. If a list of names is provided, the values will
be captured from the current environment.
Examples
--------
.. code-block:: python
print(env.env_vars)
# {}
env.add_env_vars(["VERTA_HOST"])
print(env.env_vars)
# {'VERTA_HOST': 'app.verta.ai'}
env.add_env_vars({"CUDA_VISIBLE_DEVICES": "0,1"})
print(env.env_vars)
# {'VERTA_HOST': 'app.verta.ai', 'CUDA_VISIBLE_DEVICES': '0,1'}
"""
if isinstance(env_vars, collections.Mapping):
# as mapping
env_vars_dict = dict(env_vars)
else:
# as sequence
try:
env_vars_dict = {name: os.environ[name] for name in env_vars}
except KeyError as e:
new_e = KeyError("'{}' not found in environment".format(
six.ensure_str(e.args[0]),
))
six.raise_from(new_e, None)
self._msg.environment_variables.extend(
_EnvironmentService.EnvironmentVariablesBlob(name=name, value=value)
for name, value in six.viewitems(env_vars_dict)
)
def get_attribute(self, key):
"""
Gets the attribute with name `key` from this dataset.
Parameters
----------
key : str
Name of the attribute.
Returns
-------
one of {None, bool, float, int, str}
Value of the attribute.
"""
_utils.validate_flat_key(key)
attributes = self.get_attributes()
try:
return attributes[key]
except KeyError:
six.raise_from(KeyError("no attribute found with key {}".format(key)), None)
def find(self, *args):
"""
Gets the results from this collection that match input predicates.
A predicate is a string containing a simple boolean expression consisting of:
- a dot-delimited property such as ``metrics.accuracy``
- a Python boolean operator such as ``>=``
- a literal value such as ``.8``
Parameters
----------
*args : strs
Predicates specifying results to get.
Returns
-------
The same type of object given in the input.
Examples
--------
.. code-block:: python
runs.find("hyperparameters.hidden_size == 256",
"metrics.accuracy >= .8")
# <ExperimentRuns containing 3 runs>
# alternatively:
runs.find(["hyperparameters.hidden_size == 256",
"metrics.accuracy >= .8"])
# <ExperimentRuns containing 3 runs>
"""
if len(args) == 1 and isinstance(args[0], (list, tuple)):
# to keep backward compatibility, in case user pass in a list or tuple
return self.find(*args[0])
elif not all(isinstance(predicate, six.string_types) for predicate in args):
raise TypeError("predicates must all be strings")
new_list = copy.deepcopy(self)
for predicate in args:
# split predicate
try:
key, operator, value = map(
lambda token: token.strip(),
self._OP_PATTERN.split(predicate, maxsplit=1),
)
except ValueError:
six.raise_from(
ValueError(
"predicate `{}` must be a two-operand comparison".format(
predicate
)
),
None,
)
if key.split(".")[0] not in self._VALID_QUERY_KEYS:
raise ValueError(
"key `{}` is not a valid key for querying;"
" currently supported keys are: {}".format(
key, self._VALID_QUERY_KEYS
)
)
# cast operator into protobuf enum variant
operator = self._OP_MAP[operator]
try:
value = float(value)
except ValueError: # not a number, so process as string
# maintain old behavior where input would be wrapped in quotes
if (value.startswith("'") and value.endswith("'")) or (
value.startswith('"') and value.endswith('"')
):
value = value[1:-1]
new_list._msg.predicates.append( # pylint: disable=no-member
_CommonCommonService.KeyValueQuery(
key=key,
value=_utils.python_to_val_proto(value),
operator=operator,
)
)
return new_list