def testStoreMultipleEncodings(self):
context = StorageOnlyContext()
cache = encoder.EncodingMemoryCache(context)
# This particular test needs the context to know about the cache.
context.cache = cache
my_encoder = encoder.Encoder(
context, encoder.OptionValueSet(encoder.OptionSet(),
'--parameters'))
cache.StoreEncoder(my_encoder)
videofile = encoder.Videofile('x/foo_640_480_20.yuv')
my_encoding = encoder.Encoding(my_encoder, 123, videofile)
testresult = {'foo': 'bar'}
my_encoding.result = testresult
cache.StoreEncoding(my_encoding)
my_encoding = encoder.Encoding(my_encoder, 246, videofile)
my_encoding.result = testresult
cache.StoreEncoding(my_encoding)
result = cache.AllScoredRates(my_encoder, videofile)
self.assertEquals(2, len(result))
result = cache.AllScoredEncodings(123, videofile)
self.assertEquals(1, len(result))
# Verify that it's working correctly with a new videofile object.
videofile2 = encoder.Videofile(videofile.filename)
result = cache.AllScoredEncodings(123, videofile2)
my_encoding = encoder.Encoding(my_encoder, 123, videofile2)
self.assertTrue(cache.ReadEncodingResult(my_encoding))
# Verify that it's working correctly with an encoder created via hashname.
encoder2 = encoder.Encoder(context, filename=my_encoder.Hashname())
encoding2 = encoder2.Encoding(123, videofile2)
self.assertTrue(cache.ReadEncodingResult(encoding2))
def testBrokenStoredEncoding(self):
context = StorageOnlyContext()
other_dir = os.path.join(encoder_configuration.conf.sysdir(),
'broken_files')
os.mkdir(other_dir)
cache = encoder.EncodingDiskCache(context, scoredir='broken_files')
# This particular test needs the context to know about the cache.
context.cache = cache
my_encoder = encoder.Encoder(
context, encoder.OptionValueSet(encoder.OptionSet(),
'--parameters'))
cache.StoreEncoder(my_encoder)
# Cache should start off empty.
self.assertFalse(cache.AllScoredEncodingsForEncoder(my_encoder))
videofile = encoder.Videofile('x/foo_640_480_20.yuv')
my_encoding = encoder.Encoding(my_encoder, 123, videofile)
testresult = {'foo': 'bar'}
my_encoding.result = testresult
cache.StoreEncoding(my_encoding)
# TODO(hta): Expose the filename generation as a function for testing.
with open(
os.path.join(
cache.workdir, my_encoding.encoder.Hashname(),
cache.context.codec.SpeedGroup(my_encoding.bitrate),
'%s.result' % my_encoding.videofile.basename),
'w') as scorefile:
scorefile.write('stuff that is not valid json')
result = cache.AllScoredEncodingsForEncoder(my_encoder)
self.assertFalse(result)
self.assertEquals(1, len(cache.bad_encodings))
def testEncodersInMultipleRepos(self):
test_tools.EmptyWorkDirectory()
context = StorageOnlyContext()
cache = encoder.EncodingDiskCache(context)
context.cache = cache
my_encoder = encoder.Encoder(
context, encoder.OptionValueSet(encoder.OptionSet(),
'--parameters'))
other_dir = os.path.join(encoder_configuration.conf.sysdir(),
'multirepo_test')
os.mkdir(other_dir)
other_cache = encoder.EncodingDiskCache(context,
scoredir='multirepo_test')
encoder_configuration.conf.override_scorepath_for_test([other_dir])
other_cache.StoreEncoder(my_encoder)
encoders = cache.AllEncoderFilenames(only_workdir=True)
self.assertEquals(0, len(encoders))
encoders = cache.AllEncoderFilenames(only_workdir=False)
self.assertEquals(1, len(encoders))
fetched_encoder = encoder.Encoder(context, filename=encoders[0])
self.assertEquals(my_encoder.parameters.ToString(),
fetched_encoder.parameters.ToString())
my_encoding = encoder.Encoding(
my_encoder, 123, encoder.Videofile('x/foo_640_480_20.yuv'))
testresult = {'foo': 'bar'}
my_encoding.result = testresult
other_cache.StoreEncoding(my_encoding)
# With a specified directory, we should find it in only one place.
self.assertTrue(other_cache.ReadEncodingResult(my_encoding))
# Without a specified directory, we should find it on the searchpath.
self.assertTrue(cache.ReadEncodingResult(my_encoding))
# Without a searchpath, we shouldn't find it in the default cache.
encoder_configuration.conf.override_scorepath_for_test([])
self.assertFalse(cache.ReadEncodingResult(my_encoding))
def testReadResultEncodedInAst(self):
# We've changed the storage format for results from AST to JSON.
# This test verifies that AST formatted results are still readable.
context = StorageOnlyContext()
cache = encoder.EncodingDiskCache(context)
my_encoder = encoder.Encoder(
context, encoder.OptionValueSet(encoder.OptionSet(),
'--parameters'))
cache.StoreEncoder(my_encoder)
my_encoding = encoder.Encoding(
my_encoder, 123, encoder.Videofile('x/foo_640_480_20.yuv'))
testresult = {'foo': 'bar'}
my_encoding.result = testresult
# The following code is a copy of cache.StoreEncoding, with the
# encoding step changed.
dirname = '%s/%s/%s' % (cache.workdir, my_encoding.encoder.Hashname(),
cache.context.codec.SpeedGroup(
my_encoding.bitrate))
videoname = my_encoding.videofile.basename
with open('%s/%s.result' % (dirname, videoname), 'w') as resultfile:
resultfile.write(str(my_encoding.result))
my_encoding.result = None
result = cache.ReadEncodingResult(my_encoding)
self.assertEquals(result, testresult)
def test_SuggestTweakDecreasesCq(self):
codec = vp8_cq.Vp8CodecCqMode()
videofile = encoder.Videofile('foofile_640_480_30.yuv')
my_encoder = codec.start_encoder
encoding = encoder.Encoding(my_encoder, 500, videofile)
encoding.result = { 'bitrate' : 200 }
# Since the bitrate is too high, the suggstion should be to increase it.
new_encoding = codec.SuggestTweak(encoding)
self.assertEqual('31', encoder.Option('min-q').GetValue(
new_encoding.encoder.parameters))
def AnalyzeVariants(encoding, score=False):
codec = encoding.encoder.codec
for option in codec.options:
# Get sensitivity on integer-valued options.
param_here = encoding.encoder.parameters
try:
value = int(option.GetValue(param_here))
except ValueError:
continue
# Hackish!
if value > option.min:
param_low = option.SetValue(param_here, str(value - 1))
encoding_low = encoder.Encoding(encoder.Encoder(codec, param_low),
encoding.bitrate,
encoding.videofile)
encoding_low.Recover()
if score and not encoding_low.Score():
encoding_low.Execute()
encoding_low.Store()
if encoding_low.Score():
print option.name, value, '-1', encoding_low.result,
print encoding_low.result['bitrate'] - encoding.result[
'bitrate'],
print encoding_low.result['psnr'] - encoding.result['psnr']
else:
print option.name, value, 'at lower limit'
if value < option.max:
param_high = option.SetValue(param_here, str(value + 1))
encoding_high = encoder.Encoding(
encoder.Encoder(codec, param_high), encoding.bitrate,
encoding.videofile)
encoding_high.Recover()
if score and not encoding_high.Score():
encoding_high.Execute()
encoding_high.Store()
if encoding_high.Score():
print option.name, value, '+1', encoding_high.result,
print -encoding_high.result['bitrate'] + encoding.result[
'bitrate'],
print -encoding_high.result['psnr'] + encoding.result['psnr']
else:
print option.name, value, 'at upper limit'
def test_SuggestTweakDecreasesCq(self):
codec = vp8_mpeg_1d.Vp8CodecMpeg1dMode()
videofile = encoder.Videofile('foofile_640_480_30.yuv')
my_optimizer = optimizer.Optimizer(codec)
my_encoder = codec.StartEncoder(my_optimizer.context)
encoding = encoder.Encoding(my_encoder, 500, videofile)
encoding.result = {'bitrate': 200}
# Since the bitrate is too high, the suggstion should be to increase it.
new_encoding = codec.SuggestTweak(encoding)
self.assertEqual('0',
new_encoding.encoder.parameters.GetValue('key-q'))
def test_AutoGenerateClipTime(self):
my_optimizer = optimizer.Optimizer(self.codec, self.file_set,
cache_class=self.cache_class,
score_function=ReturnsClipTime)
my_encoder = encoder.Encoder(my_optimizer.context,
encoder.OptionValueSet(encoder.OptionSet(), ''))
# Must use a tricked-out videofile to avoid disk access.
videofile = DummyVideofile('test_640x480_20.yuv', clip_time=1)
my_encoding = encoder.Encoding(my_encoder, 123, videofile)
my_encoding.result = {'psnr':42, 'bitrate':123}
# If cliptime is not present, this will raise an exception.
my_optimizer.Score(my_encoding)
def testStoreMultipleEncodings(self):
context = StorageOnlyContext()
cache = encoder.EncodingMemoryCache(context)
# This particular test needs the context to know about the cache.
context.cache = cache
my_encoder = encoder.Encoder(
context, encoder.OptionValueSet(encoder.OptionSet(),
'--parameters'))
cache.StoreEncoder(my_encoder)
videofile = encoder.Videofile('x/foo_640_480_20.yuv')
my_encoding = encoder.Encoding(my_encoder, 123, videofile)
testresult = {'foo': 'bar'}
my_encoding.result = testresult
cache.StoreEncoding(my_encoding)
my_encoding = encoder.Encoding(my_encoder, 246, videofile)
my_encoding.result = testresult
cache.StoreEncoding(my_encoding)
result = cache.AllScoredRates(my_encoder, videofile)
self.assertEquals(2, len(result))
result = cache.AllScoredEncodings(123, videofile)
self.assertEquals(1, len(result))
def SuggestTweak(self, encoding):
"""Suggest a tweak based on an encoding result.
For fixed QP, suggest increasing min-q when bitrate is too high, otherwise
suggest decreasing it.
If a parameter is already at the limit, go to the next one."""
if not encoding.result:
return None
parameters = self._SuggestTweakToName(encoding, 'key-q')
if parameters:
return encoder.Encoding(encoder.Encoder(self, parameters),
encoding.bitrate, encoding.videofile)
return None
def testStoreFetchEncoding(self):
context = StorageOnlyContext()
cache = encoder.EncodingDiskCache(context)
my_encoder = encoder.Encoder(
context, encoder.OptionValueSet(encoder.OptionSet(),
'--parameters'))
cache.StoreEncoder(my_encoder)
my_encoding = encoder.Encoding(
my_encoder, 123, encoder.Videofile('x/foo_640_480_20.yuv'))
testresult = {'foo': 'bar'}
my_encoding.result = testresult
cache.StoreEncoding(my_encoding)
my_encoding.result = None
result = cache.ReadEncodingResult(my_encoding)
self.assertEquals(result, testresult)
def testStoreMultipleEncodings(self):
# This test is sensitive to old data left around.
# The FileUsingCodecTest base class takes care of giving it an
# empty directory at test start.
context = StorageOnlyContext()
cache = encoder.EncodingDiskCache(context)
context.cache = cache # This particular test needs the link.
my_encoder = encoder.Encoder(
context, encoder.OptionValueSet(encoder.OptionSet(),
'--parameters'))
cache.StoreEncoder(my_encoder)
videofile = encoder.Videofile('x/foo_640_480_20.yuv')
my_encoding = encoder.Encoding(my_encoder, 123, videofile)
testresult = {'foo': 'bar'}
my_encoding.result = testresult
cache.StoreEncoding(my_encoding)
my_encoding = encoder.Encoding(my_encoder, 246, videofile)
my_encoding.result = testresult
cache.StoreEncoding(my_encoding)
result = cache.AllScoredRates(my_encoder, videofile)
self.assertEquals(2, len(result))
result = cache.AllScoredEncodings(123, videofile)
self.assertEquals(1, len(result))
def _SuggestTweakToName(self, encoding, name):
"""Returns a parameter string based on this encoding that has the
parameter identified by "name" changed in a way worth testing.
If no sensible change is found, returns None."""
parameters = encoding.encoder.parameters
value = int(encoder.Option(name).GetValue(parameters))
new_value = None
if encoding.result['bitrate'] > encoding.bitrate:
delta = 1
new_value = 63
candidates = range(value + 1, 64)
else:
delta = -1
new_value = 0
candidates = range(value - 1, -1, -1)
# The range of Q values is from 0 to 63.
if value + delta > 63:
print name, 'maxed out at 63'
return None # Already maxed out
if value + delta < 0:
print name, 'mined out at 0'
return None # Already at bottom
# If a previous result returned a score (which will be lower, since
# the starting point is the highest score), try the middle value
# between this and that. If none exists, go for the extreme values.
for search_value in candidates:
temp_params = encoder.Option(name).SetValue(
parameters, str(search_value))
temp_params = self.ConfigurationFixups(temp_params)
temp_encoder = encoder.Encoder(self, temp_params)
temp_encoding = encoder.Encoding(temp_encoder, encoding.bitrate,
encoding.videofile)
temp_encoding.Recover()
if temp_encoding.Score():
print name, 'found scored value', search_value
new_value = int((value + search_value) / 2)
if new_value in (value, search_value):
print name, 'already tried', value, '+1'
return None # Already tried one-step-up
break
print name, "suggesting value", new_value
parameters = encoder.Option(name).SetValue(parameters, str(new_value))
parameters = self.ConfigurationFixups(parameters)
return parameters
def testReadResultFromAlternateDir(self):
context = StorageOnlyContext()
otherdir = os.path.join(os.environ['CODEC_WORKDIR'], 'otherdir')
cache = encoder.EncodingDiskCache(context)
my_encoder = encoder.Encoder(
context, encoder.OptionValueSet(encoder.OptionSet(),
'--parameters'))
cache.StoreEncoder(my_encoder)
videofile = encoder.Videofile('x/foo_640_480_20.yuv')
my_encoding = encoder.Encoding(my_encoder, 123, videofile)
testresult = {'foo': 'bar'}
my_encoding.result = testresult
cache.StoreEncoding(my_encoding)
my_encoding.result = None
result = cache.ReadEncodingResult(my_encoding, scoredir=otherdir)
self.assertIsNone(result)
shutil.copytree(os.environ['CODEC_WORKDIR'], otherdir)
result = cache.ReadEncodingResult(my_encoding, scoredir=otherdir)
self.assertEquals(result, testresult)
def SuggestTweak(self, encoding):
"""Suggest a tweak based on an encoding result.
For fixed QP, suggest increasing min-q when bitrate is too high, otherwise
suggest decreasing it."""
if not encoding.result:
return None
if encoding.result['bitrate'] > encoding.bitrate:
delta = 1
else:
delta = -1
parameters = encoding.encoder.parameters
value = int(encoder.Option('min-q').GetValue(parameters))
# The range of min-q is from 0 to 63.
if value + delta > 63:
return None # Already maxed out
if value + delta < 0:
return None # Already at bottom
new_value = value + delta
parameters = encoder.Option('min-q').SetValue(parameters,
str(new_value))
parameters = self.ConfigurationFixups(parameters)
return encoder.Encoding(encoder.Encoder(self, parameters),
encoding.bitrate, encoding.videofile)
def testAllScoredEncodingsForEncoder(self):
context = StorageOnlyContext()
cache = encoder.EncodingDiskCache(context)
# This particular test needs the context to know about the cache.
context.cache = cache
my_encoder = encoder.Encoder(
context, encoder.OptionValueSet(encoder.OptionSet(),
'--parameters'))
cache.StoreEncoder(my_encoder)
# Cache should start off empty.
self.assertFalse(cache.AllScoredEncodingsForEncoder(my_encoder))
videofile = encoder.Videofile('x/foo_640_480_20.yuv')
my_encoding = encoder.Encoding(my_encoder, 123, videofile)
testresult = {'foo': 'bar'}
my_encoding.result = testresult
cache.StoreEncoding(my_encoding)
result = cache.AllScoredEncodingsForEncoder(my_encoder)
self.assertTrue(result)
self.assertEquals(1, len(result))
# The resulting videofile should have a basename = filename,
# because synthesizing filenames from result files loses directory
# information.
self.assertEquals('foo_640_480_20.yuv', result[0].videofile.filename)
def testReadResultFromAlternateDir(self):
context = StorageOnlyContext()
otherdir_path = os.path.join(encoder_configuration.conf.sysdir(),
'otherdir')
os.mkdir(otherdir_path)
cache = encoder.EncodingDiskCache(context)
other_cache = encoder.EncodingDiskCache(context, scoredir='otherdir')
my_encoder = encoder.Encoder(
context, encoder.OptionValueSet(encoder.OptionSet(),
'--parameters'))
cache.StoreEncoder(my_encoder)
videofile = encoder.Videofile('x/foo_640_480_20.yuv')
my_encoding = encoder.Encoding(my_encoder, 123, videofile)
testresult = {'foo': 'bar'}
my_encoding.result = testresult
cache.StoreEncoding(my_encoding)
my_encoding.result = None
result = other_cache.ReadEncodingResult(my_encoding)
self.assertIsNone(result)
shutil.rmtree(otherdir_path)
shutil.copytree(encoder_configuration.conf.workdir(), otherdir_path)
result = other_cache.ReadEncodingResult(my_encoding)
self.assertEquals(result, testresult)
def RebaseEncoding(self, encoding):
"""Take an encoding from another context and rebase it to
this context, using the same encoder arguments."""
return encoder.Encoding(self.RebaseEncoder(encoding.encoder),
encoding.bitrate, encoding.videofile)
