def _graph_blueprint_aprx_quality(self):
"""
Draw quality image graph
"""
used_files = self.config['used_files']
axis_data = []
print(used_files)
fig = plt.figure(figsize=(8, 4))
gs = gridspec.GridSpec(nrows=1, ncols=3, hspace=0.2, wspace=0.02)
files = []
utils.msg_info('graph', 'quality_graph')
"""
Fix for lossless quality
"""
for file in utils.get_path_recursive(self.config['source']):
last_file = file
"""
Opt matching according to file name
"""
cur_opt = file[file.rfind('+') + 1:file.rfind('.')]
if cur_opt in used_files:
files.append(file)
for i in range(0, len(files)):
"""
Create graph
"""
axis_data.append(fig.add_subplot(gs[0, i:i + 1]))
"""
Remove ticks
"""
axis_data[i].set_xticks([])
axis_data[i].set_yticks([])
"""
Prepare and show image
"""
img = cv2.imread(files[i])
axis_data[i].imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
"""
Set label
"""
if used_files[i] == 'lossless':
label = 'bezeztrátové'
else:
label = '{} bpp\n'.format(used_files[i])
axis_data[i].set_xlabel(label, labelpad=10)
plt.subplots_adjust(left=0.05, right=0.97, top=0.90, bottom=0.1)
self._graph_save(plt)
def _get_dir_iterator(self):
"""
Use utilitarian dir fetch as a generator that is called individually
to prevent single call exhaustion.
"""
limit = self.file_opts['limit']
include = self.file_opts['include']
exclude = self.file_opts['exclude']
file_count = 0
file_list = []
for file in utils.get_path_recursive(self.dir, defs.PPM_REGEX):
file_list.append(file)
if limit:
if limit < len(file_list):
file_list = file_list[:limit]
else:
utils.msg_info(
'recipe',
'Used limit "{}" is higher than actual file count "{}"'.
format(limit, len(file_list)))
elif exclude:
for file in file_list:
for it in exclude:
try:
file.index(it)
file_list.remove(file)
except ValueError:
continue
elif include:
filtered_file_list = []
for file in file_list:
for it in include:
try:
file.index(it)
filtered_file_list.append(file)
except ValueError:
continue
return filtered_file_list
return file_list
def _get_image_info(self, path):
"""
Read all fetched info about file
"""
ret = {}
with open(defs.EXIF_ERR, "w+") as exif_err, \
open(defs.EXIF_PATH, "w+") as exif:
return_code = subprocess.call(['exiftool', '-j', path],
stdout=exif,
stderr=exif_err)
if (return_code == 0):
with open(defs.EXIF_PATH, "r") as exif:
"""
EXIF data
"""
exif_data = json.loads(exif.read())[0]
data = {
'file': exif_data['FileName'],
'path': exif_data['Directory'],
'size': 0,
'color_depth': exif_data['MaxVal'],
'channels': 0,
'date': {},
'dims': {
'width': exif_data['ImageWidth'],
'height': exif_data['ImageHeight']
}
}
statinfo = os.stat(path).st_size
data['size'] = statinfo
"""
Determine image color depth by size comparision
"""
possible_sizes = []
dims = data['dims']['width'] * data['dims']['height']
size_in_bits = data['size'] * 8
for it in range(1, 4):
channels = it * 8
possible_sizes.append(abs(size_in_bits -
(dims * channels)))
channels = possible_sizes.index(min(possible_sizes)) + 1
if channels:
data['channels'] = channels
utils.msg_info('info read', data['file'])
ret = {
'hash':
utils.create_hash('{}/{}'.format(data['path'], data['file']),
None),
'session':
self.dir['hash'],
'timestamp':
utils.create_timestamp(),
'ctx':
data
}
os.remove(defs.EXIF_PATH)
os.remove(defs.EXIF_ERR)
return ret
def execute(self, allow_print=True):
"""
Prepare scaled file for aproximation testing
"""
self.allow_print = allow_print
self._set_file('full')
"""
Get color mode (RGB/Gray)
"""
self.color_mode = utils.get_path_file(self.file)['file'].split('.')[1]
"""
Each test can have variants that need to examined
"""
start = 0.001
end = 1
if self.mode == defs.TEST_APRX:
"""
For approximations
"""
mean_list = []
for variant in range(self.variants):
"""
Variant and variant corresponding to selected opt in
concrete case
"""
self.variant = variant
lib_variant = self.compress.get_variable_opt(self.variant)
"""
Initialization of record
"""
self._prepare_record()
self.record['testing_opt_index'] = variant
self.record['testing_opt'] = lib_variant
"""
Setting up the process
"""
utils.msg_test('routine',
'Selected variant: {}'.format(lib_variant))
"""
It's possible to have multiple values to compare againts
"""
for criterion in self.criterion['value']:
if self.criterion['type'] == 'psnr':
utils.msg_test(
'routine',
'Selected criterion: {}dB'.format(criterion))
"""
Lossless compression is selected
"""
if criterion == defs.CRITERION_LOSSLESS:
self.compress.set_quality(-1)
res = self._run_routine(lossless=True)
self._print_routine_result(res, self.testing_flag,
True)
yield self.record
else:
#print(start, end)
#print(mean_list)
if self._find_quality(start, end, criterion):
# print(start, end)
utils.msg_info('used_rate',
self.record['rate'])
"""
Store all finished rates
"""
self.aprx.append(self.record['rate'])
yield self.record
else:
yield {}
elif self.criterion['type'] == 'rate':
self.compress.set_quality(criterion)
#print(self.compress.construct(None))
res = self._run_routine(lossless=False)
self._print_routine_result(res, self.testing_flag,
True)
if self.blueprint['store']:
self._store_test_file(criterion)
yield self.record
print()
elif self.mode == defs.TEST_FINAL:
"""
Prepare record and set basic info
"""
self._prepare_record()
self.variant = self.variant_index
self.record['testing_opt_index'] = self.variant_index
"""
Match lib variant from variant index
"""
lib_variant = self.compress.get_variable_opt(self.variant_index)
self.record['testing_opt'] = lib_variant
"""
Display type info
"""
utils.msg_test('routine',
'Selected variant: {}'.format(lib_variant))
"""
Carry out compression/decompression
"""
self.compress.set_quality(self.criterion['value'])
res = self._run_routine(lossless=True)
res['quality'] = 0
self._print_routine_result(res, self.testing_flag, True)
"""
Return
"""
yield self.record
def _graph_blueprint_aprx(self):
markers = ['s', 'o', '^', 'p', 'X', 'X']
axis_data = self._graph_layout(blueprint='aprx')
"""
Two graphs side by side
"""
for index_it, graph_index in enumerate(self.fetched_data):
"""
Match data for single graph
"""
graph_data = self.fetched_data[graph_index]
"""
Aux data
"""
leg = []
colors = self._match_colors(self.graph_names[index_it])
settings = self._graph_get_settings(index_it)
utils.msg_info('graph', self.graph_names[index_it])
#print(settings)
"""
Get graph data
"""
for data_it, graph_key in enumerate(graph_data):
data_source = graph_data[graph_key]
main_axis_labels = list(data_source.keys())
main_axis_indexes = numpy.arange(0, len(main_axis_labels))
"""
Sort main labels
"""
main_axis_labels = self._process_main_axis(main_axis_labels)
auxiliary_data = []
for variant in main_axis_labels:
auxiliary_data.append(
numpy.average(data_source[variant]['value']))
main_axis_labels = self._prettify_main_axis(main_axis_labels)
selected_color = colors[-1] if graph_key == 'True' else colors[
data_it]
axis_data[index_it].plot(main_axis_labels,
auxiliary_data,
marker=markers[data_it],
linewidth=1,
color=selected_color)
leg.append('{} bpp'.format(graph_key))
"""
Set labels and titles
"""
axis_data[index_it].set_title(settings['title'])
axis_data[index_it].set_xlabel(settings['x'])
axis_data[index_it].set_ylabel(settings['y'])
axis_data[index_it].grid(alpha=0.5,
which="major",
ls="-",
axis='y')
"""
Aux settings
"""
axis_data[index_it].set_ylim(settings['limits'][0],
settings['limits'][1])
if len(leg) > 1:
axis_data[index_it].legend(leg)
axis_data[0].tick_params(axis='x', rotation=-45)
plt.subplots_adjust(left=0.05, right=0.97, top=0.90, bottom=0.175)
self._graph_save(plt)
def _graph_blueprint_test(self):
"""
Prepare individual types
"""
d = ['výkon', 'komprese', 'dekomprese']
s = ['exam', 'compress', 'decompress']
markers = ['s', 'o', '^', 'p', 'X', '+', 'x', 'D']
axis_data = self._graph_layout(blueprint=self.config['blueprint'])
"""
Get graph data
"""
for index_it, graph_index in enumerate(self.fetched_data):
graph_data = self.fetched_data[graph_index]
leg = []
try:
self.config['sessions'][index_it]['mixed_sessions']
colors = self._match_colors('mix_')
except KeyError:
colors = self._match_colors(self.graph_names[index_it])
settings = self._graph_get_settings(index_it)
utils.msg_info('graph', self.graph_names[index_it])
"""
Calculate bar offset
"""
bar_opts = len(list(graph_data.keys()))
if bar_opts > 1:
bar_offset = bar_opts / 4
for data_it, graph_key in enumerate(graph_data):
source = graph_data[graph_key]
main_axis_labels = list(source.keys())
main_axis_indexes = numpy.arange(0, len(main_axis_labels))
main_axis_labels = self._process_main_axis(main_axis_labels)
auxiliary_data = {
'value': [],
'compress': [],
'decompress': []
}
for variant in main_axis_labels:
auxiliary_data['value'].append(
numpy.average(source[variant]['value']))
"""
Normalize time for ns
"""
auxiliary_data['compress'].append(
numpy.average(source[variant]['time_compress']) *
10**9)
auxiliary_data['decompress'].append(
numpy.average(source[variant]['time_decompress']) *
10**9)
#print(auxiliary_data)
for i in range(0, 3):
key = list(auxiliary_data.keys())[i]
prepared_data = auxiliary_data[key]
"""
Normalizace for double data source graph
"""
saved_i = i
if i == 2:
i = 1
graph_type = settings[i]['type']
try:
highlighted_opt = settings[data_it]['highlighted_opt']
point = auxiliary_data[key][highlighted_opt]
axis_data[i].axhline(y=point,
color='r',
linestyle='-',
linewidth=1)
except KeyError:
pass
except IndexError:
pass
if graph_type == 'plot':
print(prepared_data)
axis_data[i].plot(main_axis_indexes, prepared_data)
elif graph_type == 'bar':
if i:
x = main_axis_indexes + ((saved_i - 1.5) * 0.15)
else:
x = main_axis_indexes
bars = axis_data[i].bar(x, prepared_data, width=0.15)
"""
Set metadata for graphs
"""
for i in range(0, 2):
sett = settings[i]
if 'legend' in sett:
axis_data[i].legend(sett['legend'])
if i > 0:
axis_data[i].set_ylim(sett['limits'][0], sett['limits'][1])
else:
if 'scale' in sett:
if sett['scale'] == 'log':
axis_data[i].set_yscale('log')
axis_data[i].set_ylim(0.05, 15)
else:
axis_data[i].set_ylim(sett['limits'][0], sett['limits'][1])
axis_data[i].set_axisbelow(True)
axis_data[i].grid(alpha=1, which="major", ls="-", axis='y')
axis_data[i].grid(alpha=0.25, which="minor", ls="dotted")
axis_data[i].set_title(sett['title'])
axis_data[i].set_xlabel(sett['x'])
axis_data[i].set_ylabel(sett['y'])
axis_data[i].set_xticks(main_axis_indexes)
if 'labels' in sett:
axis_data[i].set_xticklabels(sett['labels'])
else:
axis_data[i].set_xticklabels(main_axis_labels)
if 'rotation' in sett:
axis_data[i].tick_params(axis='x', rotation=-45)
if self.config['blueprint'] == 'test_small':
plt.subplots_adjust(left=0.075, right=0.97, top=0.90, bottom=0.15)
else:
plt.subplots_adjust(left=0.1, right=0.97, top=0.95, bottom=0.075)
self._graph_save(plt)
def _execute_routine_final(self, driver, criterion, testing_data):
#print(self.libs['compress'].construct(None))
#print(self.libs['decompress'].construct(None))
"""
We only need access to the database
"""
for item in self.db.get_many(defs.DB_TEST_DATA, {
'ctx.driver': driver,
'session': self.config['source']
}, {
'ctx': 1,
'_id': 0
}):
item = item['ctx']
self._prepare_file_test(
driver, '{}/{}'.format(item['file']['path'],
item['file']['name']))
"""
PSNR matching, discarding possible peaks in tolerance
"""
rectified_quality = self._justify_criterion_value(item['quality'])
if rectified_quality:
item['quality'] = rectified_quality
file = '{}/{}'.format(item['file']['path'], item['file']['name'])
rate_to_test = item['rate']['used']
"""
Print out info
"""
utils.msg_info(driver, file)
# utils.msg_info('rate', rate_to_test)
# utils.msg_info('opt', item['testing_opt_index'])
"""
Prepare data for routine
"""
criterion = {'type': 'rate', 'value': rate_to_test}
testing_data['variant_index'] = item['testing_opt_index']
"""
Set up routine
"""
routine = Routine(self.libs, driver, self.aux_files, file,
criterion, testing_data)
"""
Prepare items for repeated testing
"""
compression_data = []
decompression_data = []
for i in range(0, defs.REPETITON_COUNT):
res = next(routine.execute(allow_print=False))
compression_data.append(res['compression'])
decompression_data.append(res['decompression'])
"""
Set session result
"""
item['compression'] = self._aprx_test_values(compression_data)
item['decompression'] = self._aprx_test_values(decompression_data)
"""
Pack record and save it to the database
"""
self._execute_routine_result(file, item)
print()
return True
def _execute_routine_approximation(self, driver, criterion, testing_data):
file_types = []
for file in self._get_dir_iterator():
utils.msg_info(driver, file)
"""
Fetch info about this file. We want latest occurence to ensure actualness
"""
file_info = self._prepare_file_test(driver, file)
if not file_info:
return
"""
Prepare information about file to add to the record
"""
temp_file_info = file_info['ctx']
prepared_file_info = {
'type': temp_file_info['file'].split('_')[0],
'name': temp_file_info['file'],
'path': temp_file_info['path']
}
if not (prepared_file_info['type'] in file_types):
file_types.append(prepared_file_info['type'])
"""
Determine if there is special need to save testing files separetely
"""
if prepared_file_info['name'] in self.config['to_save']:
testing_data['store'] = True
else:
testing_data['store'] = False
"""
Run routine for each variant of testing param.
Get raw param to display and graph
"""
routine = Routine(self.libs, driver, self.aux_files, file,
criterion, testing_data)
for result in routine.execute(True):
if result == {}:
pass
#utils.msg_error('routine', '{} \nFile can\'t be compressed with given precision.\nMost likely not enough resolution causing peaks in criterion'.format(file))
else:
#print(result)
"""
Update with additional information. There's no need to pass
to 'routine' object even though creation would but cleaner but more
overhead in terms of time and memory would be needed
"""
result['file'] = prepared_file_info
used_rate = result['rate']
result['rate'] = {
'used':
used_rate,
'calculated':
result['size'] / (file_info['ctx']['dims']['width'] *
file_info['ctx']['dims']['height'])
}
"""
Store finished record to database
"""
self._execute_routine_result(file, result)
self.session['ctx']['file_types'] = file_types
return True
def parse(self):
"""
Parse recipe file, used for both tests and graph
"""
if not os.path.isfile(self.path):
utils.msg_error(
'recipe',
'Can\'t find recipe \n{}'.format(os.path.abspath(self.path)))
return False
with open(self.path) as json_file:
try:
data = json.load(json_file)
"""
Highest level
"""
base_level_keys = list(data.keys())
base_level_keys_len = len(base_level_keys)
if (base_level_keys_len != 1):
if (base_level_keys_len == 0):
utils.msg_info('recipe', 'Recipe empty')
else:
utils.msg_error(
'recipe',
'Recipe includes more than two base operations')
return False
"""
Highest level testing
"""
for key_item in base_level_keys:
if key_item not in list(self.required.keys()):
utils.msg_error(
'recipe',
'"{}" is not valid base operation'.format(
key_item))
return False
"""
Type of desired operation and recipe type must match
"""
if (self.type != base_level_keys[0]):
utils.msg_error(
'recipe',
'can\'t use "{}" recipe for "{}" mode'.format(
base_level_keys[0], self.type))
return False
"""
Test recipe
"""
if (self.type == 'tests'):
for test in data['tests']:
test_type = test['type']
"""
Test have two types, either approximation that do not carry final
result and final that correspond with their name
"""
if test_type == 'approximation':
test_level_keys = list(test.keys())
test_level_keys_len = len(test_level_keys)
if (test_level_keys_len < len(
self.required['tests']['core'])):
utils.msg_info('recipe', 'Recipe empty')
if not os.path.isdir(test['dir']):
utils.msg_error(
'recipe',
'Invalid dir path {}'.format(test['dir']))
return False
routines = test['routines']
for routine in routines:
if 'testing_param' in routines[routine]:
testing_param = routines[routine][
'testing_param']['flag']
if testing_param in routines[routine][
'params'].keys():
utils.msg_error(
'recipe',
'Can\'t use "{}" as a testing and regular parameter'
.format(testing_param))
return False
file_opts = test['files']
is_selected = None
for key in file_opts:
if file_opts[key]:
if (is_selected):
utils.msg_error(
'recipe',
'Too many options selected for "files", only one allowed'
)
return False
else:
is_selected = key
elif test_type == 'final':
pass
else:
utils.msg_error(
'recipe',
'Unknown test type {}'.format(test_type))
return False
self.data = data['tests']
elif (self.type == 'graph'):
self.data = data['graph']
except ValueError:
utils.msg_error(
'recipe', 'file corrupted of not valid \n{}'.format(
os.path.abspath(self.path)))
return False
return True