def _get_thread(self):
'''Run the configured number of get request. This blocks until the
put thread has inserted at least one blob so it has a blob name
to get. The blob content retrieved is verified with a checksum
by using the blob name (which contains the checksum).'''
while len(self.names) == 0:
if not self._running:
return
time.sleep(0.1)
for count in xrange(self.config['gets_per_thread']):
name = self.names[count % len(self.names)]
profile = clcommon.profile.Profile()
data = None
try:
data = self._client.get(name).read()
profile.mark_time('get')
profile.mark('get:size', len(data))
except Exception, exception:
self.log.warning(_('Get request failed: %s'), exception)
profile.mark('get:fail', 1)
self.profile_logs.append('%s\n' % profile)
if data is None:
continue
checksum = hashlib.md5(data).hexdigest() # pylint: disable=E1101
if checksum != name.split('_')[-1:][0]:
self.log.error(_('Checksum mismatch: %s'), name)
def _get_thread(self):
"""Run the configured number of get request. This blocks until the
put thread has inserted at least one blob so it has a blob name
to get. The blob content retrieved is verified with a checksum
by using the blob name (which contains the checksum)."""
while len(self.names) == 0:
if not self._running:
return
time.sleep(0.1)
for count in xrange(self.config["gets_per_thread"]):
name = self.names[count % len(self.names)]
profile = clcommon.profile.Profile()
data = None
try:
data = self._client.get(name).read()
profile.mark_time("get")
profile.mark("get:size", len(data))
except Exception, exception:
self.log.warning(_("Get request failed: %s"), exception)
profile.mark("get:fail", 1)
self.profile_logs.append("%s\n" % profile)
if data is None:
continue
checksum = hashlib.md5(data).hexdigest() # pylint: disable=E1101
if checksum != name.split("_")[-1:][0]:
self.log.error(_("Checksum mismatch: %s"), name)
def test_profile(self):
profile = clcommon.profile.Profile()
profile.mark_cpu('a')
profile.mark_time('b')
profile.reset_all()
profile.mark_all('c')
profile.mark('d', 1000000)
profile.mark('e', 99.999)
profile.mark('f', 9.9999)
profile.mark('g', 0.99999)
profile.mark('h', 0.099999)
result = str(profile).split(' ')
result.sort()
expected = ['a:cpu=0', 'b:time=0', 'c:cpu=0', 'c:time=0', 'd=1000000',
'e=100', 'f=10', 'g=1', 'h=0.1']
self.assertEquals(expected, result)
def _delete_thread(self):
'''Run the configured number of delete request. This blocks until
the put thread has inserted at least one blob so it has a blob
name to delete.'''
while len(self.names) == 0:
if not self._running:
return
time.sleep(0.1)
for count in xrange(self.config['deletes_per_thread']):
name = self.names[count % len(self.names)]
profile = clcommon.profile.Profile()
try:
self._client.delete(name)
profile.mark_time('delete')
except Exception, exception:
self.log.warning(_('Delete request failed: %s'), exception)
profile.mark('delete:fail', 1)
self.profile_logs.append('%s\n' % profile)
def _delete_thread(self):
"""Run the configured number of delete request. This blocks until
the put thread has inserted at least one blob so it has a blob
name to delete."""
while len(self.names) == 0:
if not self._running:
return
time.sleep(0.1)
for count in xrange(self.config["deletes_per_thread"]):
name = self.names[count % len(self.names)]
profile = clcommon.profile.Profile()
try:
self._client.delete(name)
profile.mark_time("delete")
except Exception, exception:
self.log.warning(_("Delete request failed: %s"), exception)
profile.mark("delete:fail", 1)
self.profile_logs.append("%s\n" % profile)
def _put_thread(self):
'''Run the configured number of put request. The checksum of the
blob content is used as the blob name so get requests can verify
the content.'''
for _count in xrange(self.config['puts_per_thread']):
num = random.randrange(2**32)
size = self.config['min_size'] + (num % self._size_range)
offset = num % max(1, self.config['max_size'] - size)
data = self._data[offset:offset + size]
name = hashlib.md5(data).hexdigest() # pylint: disable=E1101
profile = clcommon.profile.Profile()
try:
info = self._client.put(name, data)
profile.mark_time('put')
profile.mark('put:size', size)
self.names.append(info['name'])
except Exception, exception:
self.log.warning(_('Put request failed: %s'), exception)
profile.mark('put:fail', 1)
self.profile_logs.append('%s\n' % profile)
def _put_thread(self):
"""Run the configured number of put request. The checksum of the
blob content is used as the blob name so get requests can verify
the content."""
for _count in xrange(self.config["puts_per_thread"]):
num = random.randrange(2 ** 32)
size = self.config["min_size"] + (num % self._size_range)
offset = num % max(1, self.config["max_size"] - size)
data = self._data[offset : offset + size]
name = hashlib.md5(data).hexdigest() # pylint: disable=E1101
profile = clcommon.profile.Profile()
try:
info = self._client.put(name, data)
profile.mark_time("put")
profile.mark("put:size", size)
self.names.append(info["name"])
except Exception, exception:
self.log.warning(_("Put request failed: %s"), exception)
profile.mark("put:fail", 1)
self.profile_logs.append("%s\n" % profile)
def _process(self, size, image=None):
'''Process a given image size.'''
profile = clcommon.profile.Profile()
# Used image.copy originally, but that was actually much slower than
# reopening unless the image has already been modified in some way.
if image is None:
image = PIL.Image.open(StringIO.StringIO(self.raw))
profile.mark_time('open')
try:
self._load_image(image, size)
except Exception, exception:
profile.mark_time('load')
raise BadImage(_('Cannot load image (proc): %s') % exception)
profile.mark_time('load')
def _process(self, size, image=None):
'''Process a given image size.'''
profile = clcommon.profile.Profile()
# Used image.copy originally, but that was actually much slower than
# reopening unless the image has already been modified in some way.
if image is None:
image = PIL.Image.open(StringIO.StringIO(self.raw))
profile.mark_time('open')
try:
self._load_image(image, size)
except Exception, exception:
profile.mark_time('load')
raise BadImage(_('Cannot load image (proc): %s') % exception)
profile.mark_time('load')
# Used image.copy originally, but that was actually much slower than
# reopening unless the image has already been modified in some way.
if image is None:
image = PIL.Image.open(StringIO.StringIO(self.raw))
profile.mark_time('open')
try:
self._load_image(image, size)
except Exception, exception:
profile.mark_time('load')
raise BadImage(_('Cannot load image (proc): %s') % exception)
profile.mark_time('load')
width, height = size['width'], size['height']
if 'c' in size['flags']:
image = self._crop(image, width, height)
profile.mark_time('%s:crop' % size['name'])
if self._orientation > 4:
# Width and height will be reversed for these orientations.
width, height = height, width
image = image.resize((width, height), PIL.Image.ANTIALIAS)
profile.mark_time('%s:resize' % size['name'])
if self._orientation > 1:
for operation in ORIENTATION_OPERATIONS[self._orientation]:
image = image.transpose(operation)
profile.mark_time('%s:transpose' % size['name'])
if image.mode in ['P', 'LA']:
image = image.convert(mode='RGB')
profile.mark_time('%s:convert' % size['name'])
# Used image.copy originally, but that was actually much slower than
# reopening unless the image has already been modified in some way.
if image is None:
image = PIL.Image.open(StringIO.StringIO(self.raw))
profile.mark_time('open')
try:
self._load_image(image, size)
except Exception, exception:
profile.mark_time('load')
raise BadImage(_('Cannot load image (proc): %s') % exception)
profile.mark_time('load')
width, height = size['width'], size['height']
if 'c' in size['flags']:
image = self._crop(image, width, height)
profile.mark_time('%s:crop' % size['name'])
if self._orientation > 4:
# Width and height will be reversed for these orientations.
width, height = height, width
image = image.resize((width, height), PIL.Image.ANTIALIAS)
profile.mark_time('%s:resize' % size['name'])
if self._orientation > 1:
for operation in ORIENTATION_OPERATIONS[self._orientation]:
image = image.transpose(operation)
profile.mark_time('%s:transpose' % size['name'])
if image.mode in ['P', 'LA']:
image = image.convert(mode='RGB')
profile.mark_time('%s:convert' % size['name'])