def get_tensors(self, train_ids, tags, step, dims, detail):
"""
Get tensor data for given train_ids, tags, step, dims and detail.
Args:
train_ids (list): Specify list of train job ID.
tags (list): Specify list of tag.
step (int): Specify step of tag, it's necessary when detail is equal to 'data'.
dims (str): Specify dims of step, it's necessary when detail is equal to 'data'.
detail (str): Specify which data to query, available values: 'stats', 'histogram' and 'data'.
Returns:
dict, a dict including the `tensors`.
Raises:
UrlDecodeError, If unquote train id error with strict mode.
"""
Validation.check_param_empty(train_id=train_ids, tag=tags)
TensorUtils.validate_dims_format(dims)
for index, train_id in enumerate(train_ids):
try:
train_id = unquote(train_id, errors='strict')
except UnicodeDecodeError:
raise UrlDecodeError('Unquote train id error with strict mode')
else:
train_ids[index] = train_id
tensors = []
for train_id in train_ids:
tensors += self._get_train_tensors(train_id, tags, step, dims,
detail)
return {"tensors": tensors}
def get_plugins(self, train_id, manual_update=True):
"""
Queries the plug-in data for the specified training job
Args:
train_id (str): Specify a training job to query.
manual_update (bool): Specifies whether to refresh automatically.
Returns:
dict, refer to restful api.
"""
Validation.check_param_empty(train_id=train_id)
if contains_null_byte(train_id=train_id):
raise QueryStringContainsNullByteError("train job id: {} contains null byte.".format(train_id))
if manual_update:
self._data_manager.cache_train_job(train_id)
train_job = self._data_manager.get_train_job(train_id)
try:
data_visual_content = train_job.get_detail(DATAVISUAL_CACHE_KEY)
plugins = data_visual_content.get(DATAVISUAL_PLUGIN_KEY)
except exceptions.TrainJobDetailNotInCacheError:
plugins = []
if not plugins:
default_result = dict()
for plugin_name in PluginNameEnum.list_members():
default_result.update({plugin_name: list()})
return dict(plugins=default_result)
return dict(
plugins=plugins
)
def get_single_image(self, train_id, tag, step):
"""
Returns the actual image bytes for a given image.
Args:
train_id (str): The ID of the events data the image belongs to.
tag (str): The name of the tag the images belongs to.
step (int): The step of the image in the current reservoir.
Returns:
bytes, a byte string of the raw image bytes.
"""
Validation.check_param_empty(train_id=train_id, tag=tag, step=step)
step = to_int(step, "step")
tensors = self._data_manager.list_tensors(train_id, tag)
image = None
for tensor in tensors:
if tensor.step == step:
# Default value for bytes field is empty byte string normally,
# see also "Optional Fields And Default Values" in protobuf
# documentation.
image = tensor.value.encoded_image
break
if image is None:
raise ParamValueError(
"Can not find the step with given train job id and tag.")
return image
def get_single_image(self, train_id, tag, step):
"""
Returns the actual image bytes for a given image.
Args:
train_id (str): The ID of the events data the image belongs to.
tag (str): The name of the tag the images belongs to.
step (int): The step of the image in the current reservoir. If step = -1, return image of final step.
Returns:
bytes, a byte string of the raw image bytes.
"""
Validation.check_param_empty(train_id=train_id, tag=tag, step=step)
step = to_int(step, "step")
try:
tensors = self._data_manager.list_tensors(train_id, tag)
except ParamValueError as ex:
raise ImageNotExistError(ex.message)
image = _find_image(tensors, step)
if image is None:
raise ImageNotExistError(
"Can not find the step with given train job id and tag.")
return image
def _get_train_tensors(self, train_id, tags, step, dims, detail):
"""
Get tensor data for given train_id, tags, step, dims and detail.
Args:
train_id (str): Specify list of train job ID.
tags (list): Specify list of tag.
step (int): Specify step of tensor, it's necessary when detail is set to 'data'.
dims (str): Specify dims of tensor, it's necessary when detail is set to 'data'.
detail (str): Specify which data to query, available values: 'stats', 'histogram' and 'data'.
Returns:
list[dict], a list of dictionaries containing the `train_id`, `tag`, `values`.
Raises:
TensorNotExistError, If tensor with specific train_id and tag is not exist in cache.
ParamValueError, If the value of detail is not within available values:
'stats', 'histogram' and 'data'.
"""
tensors_response = []
for tag in tags:
try:
tensors = self._data_manager.list_tensors(train_id, tag)
except ParamValueError as err:
raise TensorNotExistError(err.message)
if tensors and not isinstance(tensors[0].value, TensorContainer):
raise TensorNotExistError(
"there is no tensor data in this tag: {}".format(tag))
if detail is None or detail == 'stats':
values = self._get_tensors_summary(detail, tensors)
elif detail == 'data':
Validation.check_param_empty(step=step, dims=dims)
# Limit to query max two dimensions for tensor in table view.
dims = TensorUtils.parse_shape(dims,
limit=MAX_DIMENSIONS_FOR_TENSOR)
step = to_int(step, "step")
values = self._get_tensors_data(step, dims, tensors)
elif detail == 'histogram':
values = self._get_tensors_histogram(tensors)
else:
raise ParamValueError(
'Can not support this value: {} of detail.'.format(detail))
tensor = {"train_id": train_id, "tag": tag, "values": values}
tensors_response.append(tensor)
return tensors_response
def get_tensors(self, train_ids, tags, step, dims, detail):
"""
Get tensor data for given train_ids, tags, step, dims and detail.
Args:
train_ids (list): Specify list of train job ID.
tags (list): Specify list of tag.
step (int): Specify step of tag, it's necessary when detail is equal to 'data'.
dims (str): Specify dims of step, it's necessary when detail is equal to 'data'.
detail (str): Specify which data to query, available values: 'stats', 'histogram' and 'data'.
Returns:
dict, a dict including the `tensors`.
Raises:
UrlDecodeError, If unquote train id error with strict mode.
"""
Validation.check_param_empty(train_id=train_ids, tag=tags)
if dims is not None:
if not isinstance(dims, str):
raise ParamValueError('The type of dims must be str, but got {}.'.format(type(dims)))
dims = dims.strip()
if not (dims.startswith('[') and dims.endswith(']')):
raise ParamValueError('The value: {} of dims must be '
'start with `[` and end with `]`.'.format(dims))
for dim in dims[1:-1].split(','):
dim = dim.strip()
if dim == ":":
continue
if dim.startswith('-'):
dim = dim[1:]
if not dim.isdigit():
raise ParamValueError('The value: {} of dims in the square brackets '
'must be int or `:`.'.format(dims))
for index, train_id in enumerate(train_ids):
try:
train_id = unquote(train_id, errors='strict')
except UnicodeDecodeError:
raise UrlDecodeError('Unquote train id error with strict mode')
else:
train_ids[index] = train_id
tensors = []
for train_id in train_ids:
tensors += self._get_train_tensors(train_id, tags, step, dims, detail)
return {"tensors": tensors}
def __init__(self, train_id, data_manager, tag=None):
Validation.check_param_empty(train_id=train_id)
super(GraphProcessor, self).__init__(data_manager)
train_job = self._data_manager.get_train_job_by_plugin(
train_id, PluginNameEnum.GRAPH.value)
if train_job is None:
raise exceptions.SummaryLogPathInvalid()
if not train_job['tags']:
raise ParamValueError(
"Can not find any graph data in the train job.")
if tag is None:
tag = train_job['tags'][0]
tensors = self._data_manager.list_tensors(train_id, tag=tag)
self._graph = tensors[0].value
def search_single_node(self, name):
"""
Search node by node name.
Args:
name (str): The name of node.
Returns:
dict, format is:
item_object = {'nodes': [<Node object>],
'scope_name': '',
'children': {<item_object>}}
"""
Validation.check_param_empty(name=name)
nodes = self._graph.search_single_node(name)
return nodes
def __init__(self, train_id, data_manager, tag=None):
Validation.check_param_empty(train_id=train_id)
super(GraphProcessor, self).__init__(data_manager)
train_job = self._data_manager.get_train_job_by_plugin(
train_id, PluginNameEnum.GRAPH.value)
if train_job is None:
raise exceptions.TrainJobNotExistError()
if not train_job['tags'] or (tag is not None
and tag not in train_job['tags']):
raise exceptions.GraphNotExistError()
if tag is None:
tag = train_job['tags'][0]
tensors = self._data_manager.list_tensors(train_id, tag=tag)
self._graph = tensors[0].value
def get_single_train_task(self, plugin_name, train_id):
"""
get single train task.
Args:
plugin_name (str): Plugin name, refer `PluginNameEnum`.
train_id (str): Specify a training job to query.
Returns:
{'train_jobs': list[TrainJob]}, refer to restful api.
"""
Validation.check_param_empty(plugin_name=plugin_name, train_id=train_id)
Validation.check_plugin_name(plugin_name=plugin_name)
train_job = self._data_manager.get_train_job_by_plugin(train_id=train_id, plugin_name=plugin_name)
if train_job is None:
raise exceptions.TrainJobNotExistError()
return dict(train_jobs=[train_job])
def get_histograms(self, train_id, tag):
"""
Builds a JSON-serializable object with information about histogram data.
Args:
train_id (str): The ID of the events data.
tag (str): The name of the tag the histogram data all belong to.
Returns:
dict, a dict including the `train_id`, `tag`, and `histograms'.
{
"train_id": ****,
"tag": ****,
"histograms": [{
"wall_time": ****,
"step": ****,
"bucket": [[**, **, **]],
},
{...}
]
}
"""
Validation.check_param_empty(train_id=train_id, tag=tag)
logger.info("Start to process histogram data...")
try:
tensors = self._data_manager.list_tensors(train_id, tag)
except ParamValueError as err:
raise HistogramNotExistError(err.message)
histograms = []
for tensor in tensors:
histogram = tensor.value
buckets = histogram.buckets()
histograms.append({
"wall_time": tensor.wall_time,
"step": tensor.step,
"buckets": buckets
})
logger.info("Histogram data processing is finished!")
response = {
"train_id": train_id,
"tag": tag,
"histograms": histograms
}
return response
def get_plugins(self, train_id, manual_update=True):
"""
Queries the plug-in data for the specified training job
Args:
train_id (str): Specify a training job to query.
manual_update (bool): Specifies whether to refresh automatically.
Returns:
dict, refer to restful api.
"""
Validation.check_param_empty(train_id=train_id)
train_job = self._data_manager.get_single_train_job(
train_id, manual_update=manual_update)
if not train_job:
default_result = dict()
for plugin_name in PluginNameEnum.list_members():
default_result.update({plugin_name: list()})
return dict(plugins=default_result)
return dict(plugins=train_job['tag_mapping'])
def get_metadata_list(self, train_id, tag):
"""
Builds a JSON-serializable object with information about images.
Args:
train_id (str): The ID of the events data.
tag (str): The name of the tag the images all belong to.
Returns:
list[dict], a list of dictionaries containing the `wall_time`, `step`, `width`,
and `height` for each image.
[
{
"wall_time": ****,
"step": ****,
"width": ****,
"height": ****,
},
{...}
]
"""
Validation.check_param_empty(train_id=train_id, tag=tag)
result = []
try:
tensors = self._data_manager.list_tensors(train_id, tag)
except ParamValueError as ex:
raise ImageNotExistError(ex.message)
for tensor in tensors:
# no tensor_proto in TensorEvent
(width, height) = (tensor.value.width, tensor.value.height)
result.append({
'wall_time': tensor.wall_time,
'step': tensor.step,
'width': int(width),
'height': int(height),
})
return dict(metadatas=result)
def get_metadata_list(self, train_id, tag):
"""
Builds a JSON-serializable object with information about scalars.
Args:
train_id (str): The ID of the events data.
tag (str): The name of the tag the scalars all belonging to.
Returns:
list[dict], a list of dictionaries containing the `wall_time`, `step`, `value` for each scalar.
"""
Validation.check_param_empty(train_id=train_id, tag=tag)
job_response = []
tensors = self._data_manager.list_tensors(train_id, tag)
for tensor in tensors:
job_response.append({
'wall_time': tensor.wall_time,
'step': tensor.step,
'value': tensor.value
})
return dict(metadatas=job_response)
