def testContentLength_isInBytes(self):
q = wrappers.Request(wtest.EnvironBuilder().get_environ())
r = http_util.Respond(q, '爱', 'text/plain')
self.assertEqual(r.headers.get('Content-Length'), '3')
q = wrappers.Request(wtest.EnvironBuilder().get_environ())
r = http_util.Respond(q, '爱'.encode('utf-8'), 'text/plain')
self.assertEqual(r.headers.get('Content-Length'), '3')
def testResponseCharsetTranscoding(self):
bean = '要依法治国是赞美那些谁是公义的和惩罚恶人。 - 韩非'
# input is unicode string, output is gbk string
q = wrappers.Request(wtest.EnvironBuilder().get_environ())
r = http_util.Respond(q, bean, 'text/plain; charset=gbk')
self.assertEqual(r.response[0], bean.encode('gbk'))
# input is utf-8 string, output is gbk string
q = wrappers.Request(wtest.EnvironBuilder().get_environ())
r = http_util.Respond(q, bean.encode('utf-8'),
'text/plain; charset=gbk')
self.assertEqual(r.response[0], bean.encode('gbk'))
# input is object with unicode strings, output is gbk json
q = wrappers.Request(wtest.EnvironBuilder().get_environ())
r = http_util.Respond(q, {'red': bean},
'application/json; charset=gbk')
self.assertEqual(r.response[0],
b'{"red": "' + bean.encode('gbk') + b'"}')
# input is object with utf-8 strings, output is gbk json
q = wrappers.Request(wtest.EnvironBuilder().get_environ())
r = http_util.Respond(q, {'red': bean.encode('utf-8')},
'application/json; charset=gbk')
self.assertEqual(r.response[0],
b'{"red": "' + bean.encode('gbk') + b'"}')
# input is object with gbk strings, output is gbk json
q = wrappers.Request(wtest.EnvironBuilder().get_environ())
r = http_util.Respond(q, {'red': bean.encode('gbk')},
'application/json; charset=gbk',
encoding='gbk')
self.assertEqual(r.response[0],
b'{"red": "' + bean.encode('gbk') + b'"}')
def _serve_graph(self, request):
"""Given a single run, return the graph definition in json format."""
run = request.args.get('run', None)
if run is None:
return http_util.Respond(request,
'query parameter "run" is required',
'text/plain', 400)
try:
graph = self._multiplexer.Graph(run)
except ValueError:
return http_util.Respond(request, '404 Not Found', code=404)
limit_attr_size = request.args.get('limit_attr_size', None)
if limit_attr_size is not None:
try:
limit_attr_size = int(limit_attr_size)
except ValueError:
return http_util.Respond(
request,
'query parameter `limit_attr_size` must be integer',
'text/plain', 400)
large_attrs_key = request.args.get('large_attrs_key', None)
try:
process_graph.prepare_graph_for_ui(graph, limit_attr_size,
large_attrs_key)
except ValueError as e:
return http_util.Respond(request, e.message, 'text/plain', 400)
return http_util.Respond(request, str(graph),
'text/x-protobuf') # pbtxt
def _serve_compressed_histograms(self, request):
"""Given a tag and single run, return an array of compressed histograms."""
tag = request.args.get('tag')
run = request.args.get('run')
compressed_histograms = self._multiplexer.CompressedHistograms(
run, tag)
if request.args.get('format') == _OutputFormat.CSV:
string_io = StringIO()
writer = csv.writer(string_io)
# Build the headers; we have two columns for timing and two columns for
# each compressed histogram bucket.
headers = ['Wall time', 'Step']
if compressed_histograms:
bucket_count = len(
compressed_histograms[0].compressed_histogram_values)
for i in xrange(bucket_count):
headers += [
'Edge %d basis points' % i,
'Edge %d value' % i
]
writer.writerow(headers)
for compressed_histogram in compressed_histograms:
row = [
compressed_histogram.wall_time, compressed_histogram.step
]
for value in compressed_histogram.compressed_histogram_values:
row += [value.rank_in_bps, value.value]
writer.writerow(row)
return http_util.Respond(request, string_io.getvalue(), 'text/csv')
else:
return http_util.Respond(request, compressed_histograms,
'application/json')
def testAcceptGzip_compressesResponse(self):
fall_of_hyperion_canto1_stanza1 = '\n'.join([
'Fanatics have their dreams, wherewith they weave',
'A paradise for a sect; the savage too',
'From forth the loftiest fashion of his sleep',
'Guesses at Heaven; pity these have not',
'Trac\'d upon vellum or wild Indian leaf',
'The shadows of melodious utterance.',
'But bare of laurel they live, dream, and die;',
'For Poesy alone can tell her dreams,',
'With the fine spell of words alone can save',
'Imagination from the sable charm',
'And dumb enchantment. Who alive can say,',
'\'Thou art no Poet may\'st not tell thy dreams?\'',
'Since every man whose soul is not a clod',
'Hath visions, and would speak, if he had loved',
'And been well nurtured in his mother tongue.',
'Whether the dream now purpos\'d to rehearse',
'Be poet\'s or fanatic\'s will be known',
'When this warm scribe my hand is in the grave.',
])
e1 = wtest.EnvironBuilder(headers={
'Accept-Encoding': '*'
}).get_environ()
any_encoding = wrappers.Request(e1)
r = http_util.Respond(any_encoding, fall_of_hyperion_canto1_stanza1,
'text/plain')
self.assertEqual(r.headers.get('Content-Encoding'), 'gzip')
self.assertEqual(_gunzip(r.response[0]),
fall_of_hyperion_canto1_stanza1.encode('utf-8'))
e2 = wtest.EnvironBuilder(headers={
'Accept-Encoding': 'gzip'
}).get_environ()
gzip_encoding = wrappers.Request(e2)
r = http_util.Respond(gzip_encoding, fall_of_hyperion_canto1_stanza1,
'text/plain')
self.assertEqual(r.headers.get('Content-Encoding'), 'gzip')
self.assertEqual(_gunzip(r.response[0]),
fall_of_hyperion_canto1_stanza1.encode('utf-8'))
r = http_util.Respond(any_encoding, fall_of_hyperion_canto1_stanza1,
'image/png')
self.assertEqual(r.response[0],
fall_of_hyperion_canto1_stanza1.encode('utf-8'))
def _serve_static_file(self, request, path):
"""Serves the static file located at the given path.
Args:
request: A werkzeug Request
path: The path of the static file, relative to the tensorboard/ directory.
Returns:
A werkzeug.Response application.
"""
# Strip off the leading forward slash.
orig_path = path.lstrip('/')
if not self._path_is_safe(orig_path):
logging.warning('path not safe: %s', orig_path)
return http_util.Respond(request, 'Naughty naughty!', 'text/plain',
400)
# Resource loader wants a path relative to //WORKSPACE/tensorflow.
path = os.path.join('tensorboard', orig_path)
# Open the file and read it.
try:
contents = resource_loader.load_resource(path)
except IOError:
# For compatibility with latest version of Bazel, we renamed bower
# packages to use '_' rather than '-' in their package name.
# This means that the directory structure is changed too.
# So that all our recursive imports work, we need to modify incoming
# requests to map onto the new directory structure.
path = orig_path
components = path.split('/')
components[0] = components[0].replace('-', '_')
path = ('/').join(components)
# Bazel keeps all the external dependencies in //WORKSPACE/external.
# and resource loader wants a path relative to //WORKSPACE/tensorflow/.
path = os.path.join('../external', path)
try:
contents = resource_loader.load_resource(path)
except IOError:
logging.info('path %s not found, sending 404', path)
return http_util.Respond(request,
'Not found',
'text/plain',
code=404)
mimetype, content_encoding = mimetypes.guess_type(path)
mimetype = mimetype or 'application/octet-stream'
return http_util.Respond(request,
contents,
mimetype,
expires=3600,
content_encoding=content_encoding)
def _serve_scalars(self, request):
"""Given a tag and single run, return array of ScalarEvents."""
# TODO(cassandrax): return HTTP status code for malformed requests
tag = request.args.get('tag')
run = request.args.get('run')
values = self._multiplexer.Scalars(run, tag)
if request.args.get('format') == _OutputFormat.CSV:
string_io = StringIO()
writer = csv.writer(string_io)
writer.writerow(['Wall time', 'Step', 'Value'])
writer.writerows(values)
return http_util.Respond(request, string_io.getvalue(), 'text/csv')
else:
return http_util.Respond(request, values, 'application/json')
def testHeadRequest_doesNotWrite(self):
builder = wtest.EnvironBuilder(method='HEAD')
env = builder.get_environ()
request = wrappers.Request(env)
r = http_util.Respond(request, '<b>hello world</b>', 'text/html')
self.assertEqual(r.status_code, 200)
self.assertEqual(r.response[0], six.b(''))
def _serve_run_metadata(self, request):
"""Given a tag and a TensorFlow run, return the session.run() metadata."""
tag = request.args.get('tag', None)
run = request.args.get('run', None)
if tag is None:
return http_util.Respond(
request, 'query parameter "tag" is required', 'text/plain', 400)
if run is None:
return http_util.Respond(
request, 'query parameter "run" is required', 'text/plain', 400)
try:
run_metadata = self._multiplexer.RunMetadata(run, tag)
except ValueError:
return http_util.Respond(request, '404 Not Found', code=404)
return http_util.Respond(
request, str(run_metadata), 'text/x-protobuf') # pbtxt
def _serve_runs(self, request):
"""WSGI app serving a JSON object about runs and tags.
Returns a mapping from runs to tagType to list of tags for that run.
Args:
request: A werkzeug request
Returns:
A werkzeug Response with the following content:
{runName: {images: [tag1, tag2, tag3],
audio: [tag4, tag5, tag6],
scalars: [tagA, tagB, tagC],
histograms: [tagX, tagY, tagZ],
firstEventTimestamp: 123456.789}}
"""
runs = self._multiplexer.Runs()
for run_name, run_data in runs.items():
try:
run_data[
'firstEventTimestamp'] = self._multiplexer.FirstEventTimestamp(
run_name)
except ValueError:
logging.warning(
'Unable to get first event timestamp for run %s', run_name)
run_data['firstEventTimestamp'] = None
return http_util.Respond(request, runs, 'application/json')
def _serve_scalars(self, request):
"""Given a tag and single run, return array of ScalarEvents."""
# TODO(cassandrax): return HTTP status code for malformed requests
tag = request.args.get('tag')
run = request.args.get('run')
values = self._multiplexer.Scalars(run, tag)
return http_util.Respond(request, values, 'application/json')
def _serve_individual_audio(self, request):
"""Serves an individual audio clip."""
tag = request.args.get('tag')
run = request.args.get('run')
index = int(request.args.get('index'))
audio = self._multiplexer.Audio(run, tag)[index]
return http_util.Respond(request, audio.encoded_audio_string,
audio.content_type)
def _serve_compressed_histograms(self, request):
"""Given a tag and single run, return an array of compressed histograms."""
tag = request.args.get('tag')
run = request.args.get('run')
compressed_histograms = self._multiplexer.CompressedHistograms(
run, tag)
return http_util.Respond(request, compressed_histograms,
'application/json')
def _serve_image(self, request):
"""Serves an individual image."""
tag = request.args.get('tag')
run = request.args.get('run')
index = int(request.args.get('index'))
image = self._multiplexer.Images(run, tag)[index]
encoded_image_string = image.encoded_image_string
content_type = _content_type_for_image(encoded_image_string)
return http_util.Respond(request, encoded_image_string, content_type)
def _serve_health_pills_helper(self, request):
"""Responds with health pills.
Accepts POST requests and responds with health pills. Specifically, the
handler expects a "node_names" POST data key. The value of that key should
be a JSON-ified list of node names for which the client would like to
request health pills. This data is sent via POST instead of GET because URL
length is limited.
This handler responds with a JSON-ified object mapping from node names to a
list of HealthPillEvents. Node names for which there are no health pills to
be found are excluded from the mapping.
Args:
request: The request issued by the client for health pills.
Returns:
A werkzeug BaseResponse object.
"""
if request.method != 'POST':
logging.error('%s requests are forbidden by the debugger plugin.',
request.method)
return wrappers.Response(status=405)
if _NODE_NAMES_POST_KEY not in request.form:
logging.error(
'The %s POST key was not found in the request for health pills.',
_NODE_NAMES_POST_KEY)
return wrappers.Response(status=400)
jsonified_node_names = request.form[_NODE_NAMES_POST_KEY]
try:
node_names = json.loads(jsonified_node_names)
except Exception as e: # pylint: disable=broad-except
# Different JSON libs raise different exceptions, so we just do a
# catch-all here. This problem is complicated by how Tensorboard might be
# run in many different environments, as it is open-source.
logging.error('Could not decode node name JSON string %s: %s',
jsonified_node_names, e)
return wrappers.Response(status=400)
if not isinstance(node_names, list):
logging.error('%s is not a JSON list of node names:',
jsonified_node_names)
return wrappers.Response(status=400)
# TODO(chizeng): Actually respond with the health pills per node name.
return http_util.Respond(request,
node_names,
mimetype='application/json')
def _serve_audio(self, request):
"""Given a tag and list of runs, serve a list of audio.
Note that the audio clips themselves are not sent; instead, we respond with
URLs to the audio. The frontend should treat these URLs as opaque and should
not try to parse information about them or generate them itself, as the
format may change.
Args:
request: A werkzeug.wrappers.Request object.
Returns:
A werkzeug.Response application.
"""
tag = request.args.get('tag')
run = request.args.get('run')
audio_list = self._multiplexer.Audio(run, tag)
response = self._audio_response_for_run(audio_list, run, tag)
return http_util.Respond(request, response, 'application/json')
def testHelloWorld(self):
q = wrappers.Request(wtest.EnvironBuilder().get_environ())
r = http_util.Respond(q, '<b>hello world</b>', 'text/html')
self.assertEqual(r.status_code, 200)
self.assertEqual(r.response[0], six.b('<b>hello world</b>'))
def _serve_histograms(self, request):
"""Given a tag and single run, return an array of histogram values."""
tag = request.args.get('tag')
run = request.args.get('run')
values = self._multiplexer.Histograms(run, tag)
return http_util.Respond(request, values, 'application/json')
def testPlainText_appendsUtf8ToContentType(self):
q = wrappers.Request(wtest.EnvironBuilder().get_environ())
r = http_util.Respond(q, 'hello', 'text/plain')
h = r.headers
self.assertEqual(h.get('Content-Type'), 'text/plain; charset=utf-8')
def _serve_health_pills_handler(self, request):
"""A (wrapped) werkzeug handler for serving health pills.
Accepts POST requests and responds with health pills. Specifically, the
handler expects a required "node_names" and an optional "run" POST data key.
The value of the "node_names" key should be a JSON-ified list of node names
for which the client would like to request health pills. The value of the
"run" key (which defaults to ".") should be the run to retrieve health pills
for. This data is sent via POST (not GET) because URL length is limited.
This handler responds with a JSON-ified object mapping from node names to a
list of health pill event objects, each of which has these properties.
{
'wall_time': float,
'step': int,
'node_name': string,
'output_slot': int,
# A list of 12 floats that summarizes the elements of the tensor.
'value': float[],
}
Node names for which there are no health pills to be found are excluded from
the mapping.
Args:
request: The request issued by the client for health pills.
Returns:
A werkzeug BaseResponse object.
"""
if request.method != 'POST':
logging.error('%s requests are forbidden by the debugger plugin.',
request.method)
return wrappers.Response(status=405)
if _NODE_NAMES_POST_KEY not in request.form:
logging.error(
'The %s POST key was not found in the request for health pills.',
_NODE_NAMES_POST_KEY)
return wrappers.Response(status=400)
jsonified_node_names = request.form[_NODE_NAMES_POST_KEY]
try:
node_names = json.loads(jsonified_node_names)
except Exception as e: # pylint: disable=broad-except
# Different JSON libs raise different exceptions, so we just do a
# catch-all here. This problem is complicated by how Tensorboard might be
# run in many different environments, as it is open-source.
logging.error('Could not decode node name JSON string %s: %s',
jsonified_node_names, e)
return wrappers.Response(status=400)
if not isinstance(node_names, list):
logging.error('%s is not a JSON list of node names:',
jsonified_node_names)
return wrappers.Response(status=400)
mapping = collections.defaultdict(list)
run = request.form.get(_RUN_POST_KEY, _DEFAULT_RUN)
for node_name in node_names:
try:
pill_events = self._event_multiplexer.HealthPills(
run, node_name)
for pill_event in pill_events:
mapping[node_name].append({
'wall_time': pill_event[0],
'step': pill_event[1],
'node_name': pill_event[2],
'output_slot': pill_event[3],
'value': pill_event[4],
})
except KeyError:
logging.info('No health pills found for node %s.', node_name)
return http_util.Respond(request, mapping, 'application/json')
def _serve_logdir(self, request):
"""Respond with a JSON object containing this TensorBoard's logdir."""
return http_util.Respond(request, {'logdir': self._logdir},
'application/json')
def testExpires_setsCruiseControl(self):
q = wrappers.Request(wtest.EnvironBuilder().get_environ())
r = http_util.Respond(q, '<b>hello world</b>', 'text/html', expires=60)
self.assertEqual(r.headers.get('Cache-Control'), 'private, max-age=60')
def _serve_health_pills_handler(self, request):
"""A (wrapped) werkzeug handler for serving health pills.
Accepts POST requests and responds with health pills. The request accepts
several POST parameters:
node_names: (required string) A JSON-ified list of node names for which
the client would like to request health pills.
run: (optional string) The run to retrieve health pills for. Defaults to
'.'. This data is sent via POST (not GET) since URL length is limited.
step: (optional integer): The session run step for which to
retrieve health pills. If provided, the handler reads the health pills
of that step from disk (which is slow) and produces a response with
only health pills at that step. If not provided, the handler returns a
response with health pills at all steps sampled by the event
multiplexer (the fast path). The motivation here is that, sometimes,
one desires to examine health pills at a specific step (to say find
the first step that causes a model to blow up with NaNs).
get_plugin_apps must be called before this slower feature is used
because that method passes the logdir (directory path) to this plugin.
This handler responds with a JSON-ified object mapping from node names to a
list (of size 1) of health pill event objects, each of which has these
properties.
{
'wall_time': float,
'step': int,
'node_name': string,
'output_slot': int,
# A list of 12 floats that summarizes the elements of the tensor.
'value': float[],
}
Node names for which there are no health pills to be found are excluded from
the mapping.
Args:
request: The request issued by the client for health pills.
Returns:
A werkzeug BaseResponse object.
"""
if request.method != 'POST':
logging.error(
'%s requests are forbidden by the debugger plugin.', request.method)
return wrappers.Response(status=405)
if _NODE_NAMES_POST_KEY not in request.form:
logging.error(
'The %r POST key was not found in the request for health pills.',
_NODE_NAMES_POST_KEY)
return wrappers.Response(status=400)
jsonified_node_names = request.form[_NODE_NAMES_POST_KEY]
try:
node_names = json.loads(jsonified_node_names)
except Exception as e: # pylint: disable=broad-except
# Different JSON libs raise different exceptions, so we just do a
# catch-all here. This problem is complicated by how Tensorboard might be
# run in many different environments, as it is open-source.
logging.error('Could not decode node name JSON string %r: %s',
jsonified_node_names, e)
return wrappers.Response(status=400)
if not isinstance(node_names, list):
logging.error('%r is not a JSON list of node names:',
jsonified_node_names)
return wrappers.Response(status=400)
run = request.form.get(_RUN_POST_KEY, _DEFAULT_RUN)
step_string = request.form.get(_STEP_POST_KEY, None)
if step_string is None:
# Use all steps sampled by the event multiplexer (Relatively fast).
mapping = self._obtain_sampled_health_pills(run, node_names)
else:
# Read disk to obtain the health pills for that step (Relatively slow).
# Make sure that the directory for the run exists.
# Determine the directory of events file to read.
events_directory = self._logdir
if run != _DEFAULT_RUN:
# Use the directory for the specific run.
events_directory = os.path.join(events_directory, run)
step = int(step_string)
try:
mapping = self._obtain_health_pills_at_step(
events_directory, node_names, step)
except IOError as error:
logging.error(
'Error retrieving health pills for step %d: %s', step, error)
return wrappers.Response(status=404)
# Convert event_accumulator.HealthPillEvents to JSON-able dicts.
jsonable_mapping = {}
for node_name, events in mapping.items():
jsonable_mapping[node_name] = [e._asdict() for e in events]
return http_util.Respond(request, jsonable_mapping, 'application/json')
def testJson_getsAutoSerialized(self):
q = wrappers.Request(wtest.EnvironBuilder().get_environ())
r = http_util.Respond(q, [1, 2, 3], 'application/json')
self.assertEqual(r.response[0], b'[1, 2, 3]')
