def test_webp_file_dataset(self):
"""Test case for WebPDataset.
"""
width = 400
height = 301
channel = 4
png_file = os.path.join(resource_loader.get_data_files_path(),
"testdata", "sample.png")
with open(png_file, 'rb') as f:
png_contents = f.read()
with self.cached_session():
image_op = image_ops.decode_png(png_contents, channels=channel)
image = image_op.eval()
self.assertEqual(image.shape, (height, width, channel))
filename = os.path.join(resource_loader.get_data_files_path(),
"testdata", "sample.webp")
filenames = constant_op.constant([filename], dtypes.string)
num_repeats = 2
dataset = image_dataset_ops.WebPDataset(filenames).repeat(num_repeats)
iterator = dataset.make_initializable_iterator()
init_op = iterator.initializer
get_next = iterator.get_next()
with self.cached_session() as sess:
sess.run(init_op)
for _ in range(num_repeats): # Dataset is repeated.
v = sess.run(get_next)
self.assertAllEqual(image, v)
with self.assertRaises(errors.OutOfRangeError):
sess.run(get_next)
def _loadFileAndTest(self, filename, width, height, frames, bmp_filename,
index):
"""Loads an video file and validates the output tensor.
Args:
filename: The filename of the input file.
width: The width of the video.
height: The height of the video.
frames: The frames of the video.
"""
with self.test_session():
path = os.path.join(resource_loader.get_data_files_path(),
'testdata', filename)
with open(path, 'rb') as f:
contents = f.read()
bmp_path = os.path.join(resource_loader.get_data_files_path(),
'testdata', bmp_filename)
with open(bmp_path, 'rb') as f:
bmp_contents = f.read()
image_op = image_ops.decode_bmp(bmp_contents)
image = image_op.eval()
self.assertEqual(image.shape, (height, width, 3))
video_op = ffmpeg.decode_video(contents)
video = video_op.eval()
self.assertEqual(video.shape, (frames, height, width, 3))
self.assertAllEqual(video[index, :, :, :], image)
def _loadFileAndTest(self, filename, width, height, frames, bmp_filename,
index):
"""Loads an video file and validates the output tensor.
Args:
filename: The filename of the input file.
width: The width of the video.
height: The height of the video.
frames: The frames of the video.
bmp_filename: The filename for the bmp file.
index: Index location inside the video.
"""
with self.cached_session():
path = os.path.join(resource_loader.get_data_files_path(),
'testdata', filename)
with open(path, 'rb') as f:
contents = f.read()
bmp_path = os.path.join(resource_loader.get_data_files_path(),
'testdata', bmp_filename)
with open(bmp_path, 'rb') as f:
bmp_contents = f.read()
image_op = image_ops.decode_bmp(bmp_contents)
image = image_op.eval()
self.assertEqual(image.shape, (height, width, 3))
video_op = ffmpeg.decode_video(contents)
video = video_op.eval()
self.assertEqual(video.shape, (frames, height, width, 3))
# ffmpeg produces results where channels can be off 1.
self.assertAllClose(video[index, :, :, :], image, atol=1)
def _loadFileAndTest(self, filename, width, height, frames, bmp_filename, index):
"""Loads an video file and validates the output tensor.
Args:
filename: The filename of the input file.
width: The width of the video.
height: The height of the video.
frames: The frames of the video.
"""
with self.test_session():
path = os.path.join(resource_loader.get_data_files_path(), 'testdata',
filename)
with open(path, 'rb') as f:
contents = f.read()
bmp_path = os.path.join(resource_loader.get_data_files_path(), 'testdata',
bmp_filename)
with open(bmp_path, 'rb') as f:
bmp_contents = f.read()
image_op = image_ops.decode_bmp(bmp_contents)
image = image_op.eval()
self.assertEqual(image.shape, (height, width, 3))
video_op = ffmpeg.decode_video(contents)
video = video_op.eval()
self.assertEqual(video.shape, (frames, height, width, 3))
self.assertAllEqual(video[index,:,:,:], image)
def test_sequence_file_dataset(self):
"""Test case for SequenceFileDataset.
The file is generated with `org.apache.hadoop.io.Text` for key/value.
There are 25 records in the file with the format of:
key = XXX
value = VALUEXXX
where XXX is replaced as the line number (starts with 001).
"""
filename = os.path.join(resource_loader.get_data_files_path(),
"testdata", "string.seq")
filenames = constant_op.constant([filename], dtypes.string)
num_repeats = 2
dataset = hadoop_dataset_ops.SequenceFileDataset(filenames).repeat(
num_repeats)
iterator = dataset_ops.make_initializable_iterator(dataset)
init_op = iterator.initializer
get_next = iterator.get_next()
with self.cached_session() as sess:
sess.run(init_op)
for _ in range(num_repeats): # Dataset is repeated.
for i in range(25): # 25 records.
v0 = b"%03d" % (i + 1)
v1 = b"VALUE%03d" % (i + 1)
self.assertEqual((v0, v1), sess.run(get_next))
with self.assertRaises(errors.OutOfRangeError):
sess.run(get_next)
def test_sequence_file_dataset(self):
"""Test case for SequenceFileDataset.
The file is generated with `org.apache.hadoop.io.Text` for key/value.
There are 25 records in the file with the format of:
key = XXX
value = VALUEXXX
where XXX is replaced as the line number (starts with 001).
"""
filename = os.path.join(resource_loader.get_data_files_path(),
"testdata", "string.seq")
filenames = constant_op.constant([filename], dtypes.string)
num_repeats = 2
dataset = hadoop_dataset_ops.SequenceFileDataset(filenames).repeat(
num_repeats)
iterator = dataset_ops.make_initializable_iterator(dataset)
init_op = iterator.initializer
get_next = iterator.get_next()
with self.cached_session() as sess:
sess.run(init_op)
for _ in range(num_repeats): # Dataset is repeated.
for i in range(25): # 25 records.
v0 = b"%03d" % (i + 1)
v1 = b"VALUE%03d" % (i + 1)
self.assertEqual((v0, v1), sess.run(get_next))
with self.assertRaises(errors.OutOfRangeError):
sess.run(get_next)
def testIMSLP00747_000_structure_barlines(self):
page = self.engine.run(
os.path.join(resource_loader.get_data_files_path(),
'testdata/IMSLP00747-000.png')).page[0]
self.assertEqual(len(page.system), 6)
self.assertEqual(len(page.system[0].staff), 2)
self.assertEqual(len(page.system[0].bar), 4)
self.assertEqual(len(page.system[1].staff), 2)
self.assertEqual(len(page.system[1].bar), 5)
self.assertEqual(len(page.system[2].staff), 2)
self.assertEqual(len(page.system[2].bar), 5)
self.assertEqual(len(page.system[3].staff), 2)
self.assertEqual(len(page.system[3].bar), 4)
self.assertEqual(len(page.system[4].staff), 2)
self.assertEqual(len(page.system[4].bar), 5)
self.assertEqual(len(page.system[5].staff), 2)
self.assertEqual(len(page.system[5].bar), 5)
for system in page.system:
for staff in system.staff:
self.assertEqual(staff.staffline_distance, 16)
def _loadFileAndTest(self, filename, file_format, duration_sec,
samples_per_second, channel_count):
"""Loads an audio file and validates the output tensor.
Args:
filename: The filename of the input file.
file_format: The format of the input file.
duration_sec: The duration of the audio contained in the file in seconds.
samples_per_second: The desired sample rate in the output tensor.
channel_count: The desired channel count in the output tensor.
"""
with self.test_session():
path = os.path.join(resource_loader.get_data_files_path(), 'testdata',
filename)
with open(path, 'rb') as f:
contents = f.read()
audio_op = ffmpeg.decode_audio(
contents,
file_format=file_format,
samples_per_second=samples_per_second,
channel_count=channel_count)
audio = audio_op.eval()
self.assertEqual(len(audio.shape), 2)
self.assertNear(
duration_sec * samples_per_second,
audio.shape[0],
# Duration should be specified within 10%:
0.1 * audio.shape[0])
self.assertEqual(audio.shape[1], channel_count)
def _maybe_load_sdca_ops():
with _sdca_ops_lock:
global _sdca_ops
if not _sdca_ops:
_sdca_ops = load_op_library(os.path.join(
resource_loader.get_data_files_path(), '_sdca_ops.so'))
assert _sdca_ops, 'Could not load _sdca_ops.so'
def _loadFileAndTest(self, filename, file_format, duration_sec,
samples_per_second, channel_count):
"""Loads an audio file and validates the output tensor.
Args:
filename: The filename of the input file.
file_format: The format of the input file.
duration_sec: The duration of the audio contained in the file in seconds.
samples_per_second: The desired sample rate in the output tensor.
channel_count: The desired channel count in the output tensor.
"""
with self.test_session():
path = os.path.join(resource_loader.get_data_files_path(),
'testdata', filename)
with open(path, 'rb') as f:
contents = f.read()
audio_op = ffmpeg.decode_audio(
contents,
file_format=file_format,
samples_per_second=samples_per_second,
channel_count=channel_count)
audio = audio_op.eval()
self.assertEqual(len(audio.shape), 2)
self.assertNear(
duration_sec * samples_per_second,
audio.shape[0],
# Duration should be specified within 10%:
0.1 * audio.shape[0])
self.assertEqual(audio.shape[1], channel_count)
def test_compile_and_run(self):
mlirdir = resource_loader.get_data_files_path()
for filename in os.listdir(mlirdir):
if not filename.endswith('.mlir'):
continue
with open(os.path.join(mlirdir, filename), mode='r') as f:
mlir_function = f.read()
arg_attrs = []
with ir.Context() as ctx:
ctx.allow_unregistered_dialects = True
module = ir.Module.parse(mlir_function)
func = module.body.operations[0]
function_name = ir.StringAttr(
func.attributes['sym_name']).value
if _ARG_ATTRIBUTES_NAME in func.attributes:
arg_attrs = ir.ArrayAttr(
func.attributes[_ARG_ATTRIBUTES_NAME])
logging.info(f'processing {filename}')
start = time.perf_counter()
compiled = cpurt.compile(mlir_function,
function_name,
tf_cpurt.Specialization.ENABLED,
vectorize=False)
end = time.perf_counter()
logging.info(
f'compiled {filename} in {end-start:0.4f} seconds')
if not arg_attrs:
continue
args = []
for arg_attr in arg_attrs:
attr_dict = ir.DictAttr(arg_attr)
if _SHAPE_VALUE_ATTRIBUTE_NAME in attr_dict:
shape_value_attr = ir.DenseIntElementsAttr(
attr_dict[_SHAPE_VALUE_ATTRIBUTE_NAME])
shape_value = np.array(list(shape_value_attr)).astype(
np.int32)
args.append(shape_value)
elif _STATIC_TYPE_ATTRIBUTE_NAME in attr_dict:
static_type = ir.TypeAttr(
attr_dict[_STATIC_TYPE_ATTRIBUTE_NAME]).value
shaped_type = ir.ShapedType(static_type)
dims = []
for i in range(shaped_type.rank):
dims.append(shaped_type.get_dim_size(i))
np_element_type = TfCompileAndRunTest.mlir_type_to_np_type(
shaped_type.element_type)
arg = np.random.uniform(
-10000.0, 10000.0,
size=dims).astype(np_element_type)
args.append(arg)
if len(args) != len(arg_attrs):
logging.error(
'expected valid python_test_attrs attributes for each argument'
)
continue
start = time.perf_counter()
cpurt.execute(compiled, args)
end = time.perf_counter()
logging.info(
f'executed {filename} in {end-start:0.4f} seconds')
def testBasic(self):
library_filename = os.path.join(resource_loader.get_data_files_path(),
'ackermann_op.so')
ackermann = load_library.load_op_library(library_filename)
with self.cached_session():
self.assertEqual(ackermann.ackermann().eval(), b'A(m, 0) == A(m-1, 1)')
def _maybe_load_sdca_ops():
with _sdca_ops_lock:
global _sdca_ops
if not _sdca_ops:
_sdca_ops = load_op_library(os.path.join(
resource_loader.get_data_files_path(), '_sdca_ops.so'))
assert _sdca_ops, 'Could not load _sdca_ops.so'
def testBasic(self):
library_filename = os.path.join(resource_loader.get_data_files_path(),
'duplicate_op.so')
load_library.load_op_library(library_filename)
with self.cached_session():
self.assertEqual(math_ops.add(1, 41).eval(), 42)
def testIdentical(self):
filename = six.text_type(
os.path.join(resource_loader.get_data_files_path(),
'../testdata/IMSLP00747.golden.xml'))
score = etree.fromstring(open(filename, 'rb').read())
similarity = musicxml.musicxml_similarity(score, score)
self.assertGreater(len(similarity), 1)
self.assertEqual(similarity['overall_score']['total'][0], 1.0)
def get_golden_filepath(name):
"""Returns the full path to a golden values file.
Args:
name: the name of the golden data, usually same as the test name.
"""
goldens_directory = os.path.join(_resource_loader.get_data_files_path(),
"testdata", "golden")
return os.path.join(goldens_directory, "%s.npy.golden" % name)
def testBasic(self):
library_filename = os.path.join(resource_loader.get_data_files_path(),
'duplicate_op.so')
duplicate = load_library.load_op_library(library_filename)
self.assertEqual(len(duplicate.OP_LIST.op), 0)
with self.test_session():
self.assertEqual(math_ops.add(1, 41).eval(), 42)
def test_compile_and_run(self):
filename = FLAGS.test_file_name
if not os.path.isabs(filename):
filename = os.path.join(resource_loader.get_data_files_path(), filename)
with gfile.GFile(filename, mode='r') as f:
mlir_function = f.read()
arg_attrs = []
with ir.Context() as ctx:
ctx.allow_unregistered_dialects = True
module = ir.Module.parse(mlir_function)
func = module.body.operations[0]
function_name = ir.StringAttr(func.attributes['sym_name']).value
# If the function has arguments, we expect argument attributes.
if func.regions[0].blocks[0].arguments:
self.assertIn(_ARG_ATTRIBUTES_NAME, func.attributes)
arg_attrs = ir.ArrayAttr(func.attributes[_ARG_ATTRIBUTES_NAME])
logging.info(f'processing {filename}')
start = time.perf_counter()
compiled = jitrt.compile(
mlir_function,
function_name,
tf_jitrt.Specialization.ENABLED,
vectorize=FLAGS.vectorize)
end = time.perf_counter()
logging.info(f'compiled {filename} in {end-start:0.4f} seconds')
np.random.seed(FLAGS.input_data_seed)
args = []
for arg_attr in arg_attrs:
attr_dict = ir.DictAttr(arg_attr)
if _SHAPE_VALUE_ATTRIBUTE_NAME in attr_dict:
shape_value_attr = ir.DenseIntElementsAttr(
attr_dict[_SHAPE_VALUE_ATTRIBUTE_NAME])
shape_value = np.array(list(shape_value_attr)).astype(np.int32)
args.append(shape_value)
elif _STATIC_TYPE_ATTRIBUTE_NAME in attr_dict:
static_type = ir.TypeAttr(
attr_dict[_STATIC_TYPE_ATTRIBUTE_NAME]).value
shaped_type = ir.ShapedType(static_type)
np_element_type = CompileAndRunTest.mlir_type_to_np_type(
shaped_type.element_type)
arg = np.random.uniform(
-10000.0, 10000.0, size=shaped_type.shape).astype(np_element_type)
args.append(arg)
self.assertEqual(len(args), len(arg_attrs))
start = time.perf_counter()
result = jitrt.execute(compiled, args)
end = time.perf_counter()
logging.info(f'executed {filename} in {end-start:0.4f} seconds')
if FLAGS.compare_with_tensorflow:
start = time.perf_counter()
expected = tfrt_fallback.run_tfrt_fallback(mlir_function, function_name,
args)
end = time.perf_counter()
logging.info(
f'executed {filename} via tfrt fallback in {end-start:0.4f} seconds'
)
np.testing.assert_allclose(result, expected, rtol=1e-5, atol=1e-5)
def testBasic(self):
library_filename = os.path.join(resource_loader.get_data_files_path(),
'duplicate_op.so')
duplicate = load_library.load_op_library(library_filename)
self.assertEqual(len(duplicate.OP_LIST.op), 0)
with self.test_session():
self.assertEqual(math_ops.add(1, 41).eval(), 42)
def test_compile_and_run(self):
filename = FLAGS.test_file_name
if not os.path.isabs(filename):
filename = os.path.join(resource_loader.get_data_files_path(),
filename)
with gfile.GFile(filename, mode='r') as f:
mlir_function = f.read()
arg_attrs = []
with ir.Context() as ctx:
ctx.allow_unregistered_dialects = True
module = ir.Module.parse(mlir_function)
func = module.body.operations[0]
function_name = ir.StringAttr(
func.attributes['sym_name']).value
if _ARG_ATTRIBUTES_NAME in func.attributes:
arg_attrs = ir.ArrayAttr(
func.attributes[_ARG_ATTRIBUTES_NAME])
logging.info(f'processing {filename}')
start = time.perf_counter()
compiled = jitrt.compile(mlir_function,
function_name,
tf_jitrt.Specialization.ENABLED,
vectorize=FLAGS.vectorize)
end = time.perf_counter()
logging.info(f'compiled {filename} in {end-start:0.4f} seconds')
if not arg_attrs:
return
np.random.seed(FLAGS.input_data_seed)
args = []
for arg_attr in arg_attrs:
attr_dict = ir.DictAttr(arg_attr)
if _SHAPE_VALUE_ATTRIBUTE_NAME in attr_dict:
shape_value_attr = ir.DenseIntElementsAttr(
attr_dict[_SHAPE_VALUE_ATTRIBUTE_NAME])
shape_value = np.array(list(shape_value_attr)).astype(
np.int32)
args.append(shape_value)
elif _STATIC_TYPE_ATTRIBUTE_NAME in attr_dict:
static_type = ir.TypeAttr(
attr_dict[_STATIC_TYPE_ATTRIBUTE_NAME]).value
shaped_type = ir.ShapedType(static_type)
np_element_type = CompileAndRunTest.mlir_type_to_np_type(
shaped_type.element_type)
arg = np.random.uniform(
-10000.0, 10000.0,
size=shaped_type.shape).astype(np_element_type)
args.append(arg)
if len(args) != len(arg_attrs):
logging.error(
'expected valid python_test_attrs attributes for each argument'
)
return
start = time.perf_counter()
jitrt.execute(compiled, args)
end = time.perf_counter()
logging.info(f'executed {filename} in {end-start:0.4f} seconds')
def testMusicXML(self):
filename = os.path.join(resource_loader.get_data_files_path(),
'testdata/IMSLP00747-000.png')
score = self.engine.run([filename])
num_measures = sum(
len(system.bar) - 1 for page in score.page for system in page.system)
musicxml = etree.fromstring(conversions.score_to_musicxml(score))
self.assertEqual(2, len(musicxml.findall('part')))
self.assertEqual(num_measures,
len(musicxml.find('part[1]').findall('measure')))
def testBasic(self):
library_filename = os.path.join(resource_loader.get_data_files_path(),
'ackermann_op.so')
ackermann = load_library.load_op_library(library_filename)
self.assertEqual(len(ackermann.OP_LIST.op), 1)
self.assertEqual(ackermann.OP_LIST.op[0].name, 'Ackermann')
with self.cached_session():
self.assertEqual(ackermann.ackermann().eval(), b'A(m, 0) == A(m-1, 1)')
def testProcessImage(self):
pil_image = Image.open(
os.path.join(resource_loader.get_data_files_path(),
'testdata/IMSLP00747-000.png')).convert('L')
arr = np.array(pil_image.getdata(), np.uint8).reshape(
# Size is (width, height).
pil_image.size[1],
pil_image.size[0])
page = self.engine.process_image(arr)
self.assertEqual(6, len(page.system))
def testBasic(self):
library_filename = os.path.join(resource_loader.get_data_files_path(),
'ackermann_op.so')
ackermann = load_library.load_op_library(library_filename)
self.assertEqual(len(ackermann.OP_LIST.op), 1)
self.assertEqual(ackermann.OP_LIST.op[0].name, 'Ackermann')
with self.test_session():
self.assertEqual(ackermann.ackermann().eval(),
b'A(m, 0) == A(m-1, 1)')
def _InitPathConstants():
global _API_GOLDEN_FOLDER_V1
global _API_GOLDEN_FOLDER_V2
root_golden_path_v2 = os.path.join(resource_loader.get_data_files_path(),
'..', 'golden', 'v2', 'tensorflow.pbtxt')
if FLAGS.update_goldens:
root_golden_path_v2 = os.path.realpath(root_golden_path_v2)
# Get API directories based on the root golden file. This way
# we make sure to resolve symbolic links before creating new files.
_API_GOLDEN_FOLDER_V2 = os.path.dirname(root_golden_path_v2)
_API_GOLDEN_FOLDER_V1 = os.path.normpath(
os.path.join(_API_GOLDEN_FOLDER_V2, '..', 'v1'))
def testNoteSequence(self):
filename = os.path.join(resource_loader.get_data_files_path(),
'testdata/IMSLP00747-000.png')
notes = self.engine.run(filename, output_notesequence=True)
# TODO(ringw): Fix the extra note that is detected before the actual
# first eighth note.
self.assertEqual(librosa.note_to_midi('C4'), notes.notes[1].pitch)
self.assertEqual(librosa.note_to_midi('D4'), notes.notes[2].pitch)
self.assertEqual(librosa.note_to_midi('E4'), notes.notes[3].pitch)
self.assertEqual(librosa.note_to_midi('F4'), notes.notes[4].pitch)
self.assertEqual(librosa.note_to_midi('D4'), notes.notes[5].pitch)
self.assertEqual(librosa.note_to_midi('E4'), notes.notes[6].pitch)
self.assertEqual(librosa.note_to_midi('C4'), notes.notes[7].pitch)
def _loadFileAndTest(self,
filename,
file_format,
duration_sec,
samples_per_second,
channel_count,
samples_per_second_tensor=None,
feed_dict=None,
stream=None):
"""Loads an audio file and validates the output tensor.
Args:
filename: The filename of the input file.
file_format: The format of the input file.
duration_sec: The duration of the audio contained in the file in seconds.
samples_per_second: The desired sample rate in the output tensor.
channel_count: The desired channel count in the output tensor.
samples_per_second_tensor: The value to pass to the corresponding
parameter in the instantiated `decode_audio` op. If not
provided, will default to a constant value of
`samples_per_second`. Useful for providing a placeholder.
feed_dict: Used when evaluating the `decode_audio` op. If not
provided, will be empty. Useful when providing a placeholder for
`samples_per_second_tensor`.
stream: A string specifying which stream from the content file
should be decoded. The default value is '' which leaves the
decision to ffmpeg.
"""
if samples_per_second_tensor is None:
samples_per_second_tensor = samples_per_second
with self.cached_session():
path = os.path.join(resource_loader.get_data_files_path(),
'testdata', filename)
with open(path, 'rb') as f:
contents = f.read()
audio_op = ffmpeg.decode_audio(
contents,
file_format=file_format,
samples_per_second=samples_per_second_tensor,
channel_count=channel_count,
stream=stream)
audio = audio_op.eval(feed_dict=feed_dict or {})
self.assertEqual(len(audio.shape), 2)
self.assertNear(
duration_sec * samples_per_second,
audio.shape[0],
# Duration should be specified within 10%:
0.1 * audio.shape[0])
self.assertEqual(audio.shape[1], channel_count)
def testDivisions_golden(self):
"""Test that <divisions> is propagated across all measures."""
filename = six.text_type(
os.path.join(resource_loader.get_data_files_path(),
'../testdata/IMSLP00747.golden.xml'))
score = etree.fromstring(open(filename, 'rb').read())
part_staves = musicxml.PartStaves(score)
self.assertEqual(part_staves.num_partstaves(), 2)
self.assertEqual(part_staves.num_measures(0), 22)
self.assertEqual(part_staves.num_measures(1), 22)
for i in moves.xrange(2):
for j in moves.xrange(22):
measure = part_staves.get_measure(i, j)
self.assertEqual(
measure.find('attributes/divisions').text, '8')
def testRoundTrip(self):
"""Fabricates some audio, creates a wav file, reverses it, and compares."""
with self.test_session():
path = os.path.join(
resource_loader.get_data_files_path(), 'testdata/mono_10khz.wav')
with open(path, 'r') as f:
original_contents = f.read()
audio_op = ffmpeg.decode_audio(
original_contents, file_format='wav', samples_per_second=10000,
channel_count=1)
encode_op = ffmpeg.encode_audio(
audio_op, file_format='wav', samples_per_second=10000)
encoded_contents = encode_op.eval()
self.assertEqual(original_contents, encoded_contents)
def get_default_assets_zip_provider():
"""Opens stock TensorBoard web assets collection.
Returns:
Returns function that returns a newly opened file handle to zip file
containing static assets for stock TensorBoard, or None if webfiles.zip
could not be found. The value the callback returns must be closed. The
paths inside the zip file are considered absolute paths on the web server.
"""
path = os.path.join(resource_loader.get_data_files_path(), os.pardir,
'webfiles.zip')
if not os.path.exists(path):
logging.warning('webfiles.zip static assets not found: %s', path)
return None
return lambda: open(path, 'rb')
def testRoundTrip(self):
"""Reads a wav file, writes it, and compares them."""
with self.test_session():
path = os.path.join(
resource_loader.get_data_files_path(), 'testdata/mono_10khz.wav')
with open(path, 'rb') as f:
original_contents = f.read()
audio_op = ffmpeg.decode_audio(
original_contents, file_format='wav', samples_per_second=10000,
channel_count=1)
encode_op = ffmpeg.encode_audio(
audio_op, file_format='wav', samples_per_second=10000)
encoded_contents = encode_op.eval()
self._compareWavFiles(original_contents, encoded_contents)
def testRoundTrip(self):
"""Fabricates some audio, creates a wav file, reverses it, and compares."""
with self.test_session():
path = os.path.join(resource_loader.get_data_files_path(),
'testdata/mono_10khz.wav')
with open(path, 'r') as f:
original_contents = f.read()
audio_op = ffmpeg.decode_audio(original_contents,
file_format='wav',
samples_per_second=10000,
channel_count=1)
encode_op = ffmpeg.encode_audio(audio_op,
file_format='wav',
samples_per_second=10000)
encoded_contents = encode_op.eval()
self.assertEqual(original_contents, encoded_contents)
def testBeams_sixteenthNotes(self):
filename = os.path.join(resource_loader.get_data_files_path(),
'testdata/IMSLP00747-000.png')
notes = self.engine.run([filename], output_notesequence=True)
def _sixteenth_note(pitch, start_time):
return music_pb2.NoteSequence.Note(
pitch=librosa.note_to_midi(pitch),
start_time=start_time,
end_time=start_time + 0.25)
# TODO(ringw): Fix the phantom quarter note detected before the treble
# clef, and the eighth rest before the first note (should be sixteenth).
self.assertIn(_sixteenth_note('C4', 1.5), notes.notes)
self.assertIn(_sixteenth_note('D4', 1.75), notes.notes)
self.assertIn(_sixteenth_note('E4', 2), notes.notes)
self.assertIn(_sixteenth_note('F4', 2.25), notes.notes)
def _loadFileAndTest(self, filename, file_format, duration_sec,
samples_per_second, channel_count,
samples_per_second_tensor=None, feed_dict=None,
stream=None):
"""Loads an audio file and validates the output tensor.
Args:
filename: The filename of the input file.
file_format: The format of the input file.
duration_sec: The duration of the audio contained in the file in seconds.
samples_per_second: The desired sample rate in the output tensor.
channel_count: The desired channel count in the output tensor.
samples_per_second_tensor: The value to pass to the corresponding
parameter in the instantiated `decode_audio` op. If not
provided, will default to a constant value of
`samples_per_second`. Useful for providing a placeholder.
feed_dict: Used when evaluating the `decode_audio` op. If not
provided, will be empty. Useful when providing a placeholder for
`samples_per_second_tensor`.
stream: A string specifying which stream from the content file
should be decoded. The default value is '' which leaves the
decision to ffmpeg.
"""
if samples_per_second_tensor is None:
samples_per_second_tensor = samples_per_second
with self.test_session():
path = os.path.join(resource_loader.get_data_files_path(), 'testdata',
filename)
with open(path, 'rb') as f:
contents = f.read()
audio_op = ffmpeg.decode_audio(
contents,
file_format=file_format,
samples_per_second=samples_per_second_tensor,
channel_count=channel_count, stream=stream)
audio = audio_op.eval(feed_dict=feed_dict or {})
self.assertEqual(len(audio.shape), 2)
self.assertNear(
duration_sec * samples_per_second,
audio.shape[0],
# Duration should be specified within 10%:
0.1 * audio.shape[0])
self.assertEqual(audio.shape[1], channel_count)
def load_evaluation_images(image_size):
"""Loads images for evaluation.
Args:
image_size: int. Image size.
Returns:
Tensor. A batch of evaluation images.
Raises:
IOError: If no evaluation images can be found.
"""
glob = os.path.join(resource_loader.get_data_files_path(),
_EVALUATION_IMAGES_GLOB)
evaluation_images = tf.gfile.Glob(glob)
if not evaluation_images:
raise IOError('No evaluation images found')
return tf.concat(
0, [load_image(path, image_size) for path in evaluation_images])
def test_parquet_dataset(self):
"""Test case for ParquetDataset.
Note: The sample file is generated from:
`parquet-cpp/examples/low-level-api/reader_writer`
This test extracts columns of [0, 1, 2, 4, 5]
with column data types of [bool, int32, int64, float, double].
Please check `parquet-cpp/examples/low-level-api/reader-writer.cc`
to find details of how records are generated:
Column 0 (bool): True for even rows and False otherwise.
Column 1 (int32): Equal to row_index.
Column 2 (int64): Equal to row_index * 1000 * 1000 * 1000 * 1000.
Column 4 (float): Equal to row_index * 1.1.
Column 5 (double): Equal to row_index * 1.1111111.
"""
filename = os.path.join(resource_loader.get_data_files_path(),
'testdata/parquet_cpp_example.parquet')
filenames = constant_op.constant([filename], dtypes.string)
columns = [0, 1, 2, 4, 5]
output_types = (dtypes.bool, dtypes.int32, dtypes.int64,
dtypes.float32, dtypes.float64)
num_repeats = 2
dataset = parquet_dataset_ops.ParquetDataset(
filenames, columns, output_types).repeat(num_repeats)
iterator = dataset.make_initializable_iterator()
init_op = iterator.initializer
get_next = iterator.get_next()
with self.test_session() as sess:
sess.run(init_op)
for _ in range(num_repeats): # Dataset is repeated.
for i in range(500): # 500 rows.
v0 = True if ((i % 2) == 0) else False
v1 = i
v2 = i * 1000 * 1000 * 1000 * 1000
v4 = 1.1 * i
v5 = 1.1111111 * i
self.assertAllClose((v0, v1, v2, v4, v5),
sess.run(get_next))
with self.assertRaises(errors.OutOfRangeError):
sess.run(get_next)
def test_compile(self):
mlirdir = os.path.join(resource_loader.get_data_files_path(),
'regression_tests')
for filename in os.listdir(mlirdir):
with open(os.path.join(mlirdir, filename), mode='r') as f:
mlir_function = f.read()
logging.info(f'processing {filename}')
start = time.perf_counter()
cpurt.compile(
mlir_function,
# TODO(akuegel): Currently we assume that the function is always
# named 'test'. Better would be to process the IR and get the name
# from there.
'test',
tf_cpurt.Specialization.ENABLED,
vectorize=False)
end = time.perf_counter()
logging.info(
f'compiled {filename} in {end-start:0.4f} seconds')
def test_video_dataset(self):
"""Test case for VideoDataset."""
filename = os.path.join(resource_loader.get_data_files_path(),
"testdata", "small.mp4")
filenames = constant_op.constant([filename], dtypes.string)
num_repeats = 2
dataset = video_dataset_ops.VideoDataset(filenames).repeat(num_repeats)
iterator = dataset.make_initializable_iterator()
init_op = iterator.initializer
get_next = iterator.get_next()
with self.cached_session() as sess:
sess.run(init_op)
for _ in range(num_repeats): # Dataset is repeated.
for _ in range(166): # 166 frames
v = sess.run(get_next)
self.assertAllEqual(v.shape, (320, 560, 3))
with self.assertRaises(errors.OutOfRangeError):
sess.run(get_next)
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import state_ops
from tensorflow.python.ops import gen_state_ops
from tensorflow.python.platform import resource_loader
from tensorflow.user_ops import floaty_ops
try:
# Try to load the GPU-enabled library first.
_module = load_op_library("/tf-dev/bazel-bin/tensorflow/user_ops/libthin_stack_ops_impl_gpu.so")
except Exception, e:
print(e)
# Load CPU only.
_module = load_op_library(os.path.join(resource_loader.get_data_files_path(), "thin_stack_ops_impl.so"))
thin_stack_lookup = _module.thin_stack_lookup
_thin_stack_lookup_gradient_impl = _module.thin_stack_lookup_grad
thin_stack_update = _module.thin_stack_update
@ops.RegisterShape("ThinStackLookup")
def _thin_stack_lookup_shape(op):
batch_size = op.inputs[3].get_shape()[0]
model_dim = op.inputs[0].get_shape()[1]
embedding_dim = op.inputs[1].get_shape()[1]
stack_el_shape = TensorShape((batch_size, model_dim))
buf_el_shape = TensorShape((batch_size, embedding_dim))
stack2_ptrs_shape = TensorShape((batch_size,))
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os.path
from tensorflow.python.framework import ops
from tensorflow.python.framework.load_library import load_op_library
from tensorflow.python.framework.ops import convert_to_tensor
from tensorflow.python.framework.ops import name_scope
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops.nn import sigmoid_cross_entropy_with_logits
from tensorflow.python.platform import resource_loader
_sdca_ops = load_op_library(os.path.join(resource_loader.get_data_files_path(),
'_sdca_ops.so'))
__all__ = ['SdcaModel']
class SdcaModel(object):
"""Stochastic dual coordinate ascent solver for linear models.
This class currently only supports a single machine (multi-threaded)
implementation. We expect the data, and weights to fit in a single machine.
Loss functions supported:
* Binary logistic loss
This class defines an optimizer API to train a linear model.
def setUp(self):
file_system_library = os.path.join(resource_loader.get_data_files_path(),
"test_file_system.so")
load_library.load_file_system_library(file_system_library)
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Ignite File System for checkpointing and communication with TensorBoard.
Apache Ignite is a memory-centric distributed database, caching, and
processing platform for transactional, analytical, and streaming workloads,
delivering in-memory speeds at petabyte scale. In addition to database
functionality Apache Ignite provides a distributed file system called
IGFS (https://ignite.apache.org/features/igfs.html). IGFS delivers a similar
functionality to Hadoop HDFS, but only in-memory. In fact, in addition to
its own APIs, IGFS implements Hadoop FileSystem API and can be transparently
plugged into Hadoop or Spark deployments. This contrib package contains an
integration between IGFS and TensorFlow.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
from tensorflow.contrib.ignite.python.ops import ignite_op_loader # pylint: disable=unused-import
from tensorflow.python.framework import load_library
from tensorflow.python.platform import resource_loader
file_system_library = os.path.join(resource_loader.get_data_files_path(),
"../../_ignite_ops.so")
load_library.load_file_system_library(file_system_library)
def setUp(self):
super(EncodeAudioOpTest, self).setUp()
path = os.path.join(resource_loader.get_data_files_path(),
'testdata/mono_10khz.wav')
with open(path, 'rb') as f:
self._contents = f.read()
def setUp(self):
self.midi_simple_filename = os.path.join(
resource_loader.get_data_files_path(), '../testdata/example.mid')
self.midi_complex_filename = os.path.join(
resource_loader.get_data_files_path(),
'../testdata/example_complex.mid')
