def datasetClassExampleImages(dataset_name,
class_name,
number_of_images=12,
api=None):
pairs_folder_name = 'pairs_' + dataset_name
dataset_specific_api_name = api or dataset_name
dataset_specific_api = getDatasetSpecificApi(dataset_specific_api_name)
pairs = Pairs(pairs_folder_name,
get_classes_function=dataset_specific_api.getClasses)
showFromClass(pairs, class_name, number_of_images, 1, number_of_images)
def datasetClassesExampleImages(dataset_name, number_of_images=12, api=None):
pairs_folder_name = 'pairs_' + dataset_name
dataset_specific_api_name = api or dataset_name
dataset_specific_api = getDatasetSpecificApi(dataset_specific_api_name)
pairs = Pairs(pairs_folder_name,
get_classes_function=dataset_specific_api.getClasses)
classes = pairs.getClasses()
for class_name in classes:
print('Примеры изображений из класса "' + class_name + '":')
showFromClass(pairs,
class_name,
number_of_images,
rows=1,
columns=number_of_images)
def datasetMetrics(dataset_name,
metric_name,
log_plots=False,
api=None,
examples_amount=0,
rows=None,
columns=None,
vertical=True,
max_columns=20):
examples_amount = None if (examples_amount == 'all') else examples_amount
report_file_name = 'on_flow_by_classes_' + dataset_name + '_test.json'
report = load_json_from_file(report_file_name)
pairs_folder_name = 'pairs_' + dataset_name
dataset_specific_api_name = api or dataset_name
dataset_specific_api = getDatasetSpecificApi(dataset_specific_api_name)
if (examples_amount == None) or (examples_amount > 0):
pairs = Pairs(pairs_folder_name,
get_classes_function=dataset_specific_api.getClasses)
for filter_as_string in report:
print('Для изображений',
presentFilter(json.loads(filter_as_string)) + ':')
report_for_filter = report[filter_as_string]
metrics_for_class = report_for_filter[metric_name]
for metric in metrics_for_class:
if metric['name'] == 'distribution':
print('Распределение:')
preprocess = (lambda x: log(x)
if x != 0 else 0) if log_plots else (lambda x: x)
if len(metric['value']) > max_columns:
scaleFactor = ceil(len(metric['value']) / max_columns)
else:
scaleFactor = 1
plotDistribution(metric['value'],
preprocess,
vertical=vertical,
scaleFactor=scaleFactor)
for interval in metric['value']:
if (examples_amount == None) or (examples_amount > 0):
if 'examples' in interval:
print('Примеры для интервала с', interval['from'],
'до', interval['to'], ':')
if len(interval['examples'][0]) == 1:
example_pairs = [
pairs[example[0]] for example in
interval['examples'][:examples_amount]
]
showPairsImages(example_pairs)
else:
for example in interval[
'examples'][:examples_amount]:
example_pairs = [pairs[i] for i in example]
showPairsImages(example_pairs)
elif (metric['name'] == 'kmeans') and ((examples_amount == None) or
(examples_amount > 0)):
print('Результаты кластеризации на', len(metric['value']),
'кластер' + properEnding(len(metric['value'])) + ':')
for cluster_id in metric['value']:
print('Примеры изображений из кластера ' + cluster_id +
':')
example_pairs = [
pairs[object_id] for object_id in metric['value']
[cluster_id][:examples_amount]
]
if len(example_pairs) > 0:
printPairsPaths(example_pairs)
elif metric['name'] == 'mean':
if metric_name == 'mean_histogram':
plt.hist(metric['value'], bins=len(metric['value']))
plt.show()
else:
print('Среднее значение:', metric['value'])
default='1')
args = parser.parse_args()
dataset_name = args.dataset
dataset_specific_api_name = args.api or dataset_name
threads = int(args.threads)
overwrite = int(args.overwrite)
continue_calc = int(args.continue_calc)
# creating report object
report_file_path = 'report_' + dataset_name + '.json'
if overwrite or (not os.path.exists(report_file_path)):
report = Report(report_file_path)
# geting dataset specific api
dataset_specific_api = getDatasetSpecificApi(dataset_specific_api_name)
# geting pairs from directory
directory = 'pairs_' + dataset_name + '_new'
if overwrite or (not os.path.exists(directory)) or continue_calc:
directory = directory.replace('_new', '')
pairs = Pairs(directory, get_classes_function=dataset_specific_api.getClasses)
# using this function you can see and list (press q) images with from class
# showFromClassContinuously(pairs, 'Unknown', dataset_specific_api.getClasses)
# counting objects in classes
if overwrite or (not os.path.exists(report_file_path)):
objects_number_by_class = pairs.countObjectsInClasses()
report.write('objects number by class', objects_number_by_class)
from tsnecuda import TSNE
import numpy as np
from pairs import Pairs
from filters import getPairsMetrics, getPairsMetricsWithPairs
from dataset_specific_api import getDatasetSpecificApi
import cProfile
from objects import getObjectPath
dataset_name = 'faces_not_clear_2'
clusters_number = 2
dataset_dir = 'pairs_' + dataset_name + '_new'
dataset_specific_api = getDatasetSpecificApi('experiment')
pairs = Pairs(dataset_dir,
get_classes_function=dataset_specific_api.getClasses)
# pairs.filterBy({'__class__': ['happy']})
pairs.setReturnNames(True)
hogs_and_pairs = list(getPairsMetricsWithPairs(pairs, 'hog', threads=24))
pairs.setReturnNames(False)
hogs = np.array([e[0] for e in hogs_and_pairs])
print(hogs.shape[0], 'elements')
tsne = TSNE(n_components=2).fit_transform(hogs)
print(tsne)
overwrite = int(args.overwrite)
splited = int(args.splited)
# geting files
print('geting files...')
files = getFiles(dataset_folder)
print('got', len(files), 'files')
# geting dataset specific api
dataset_specific_api = getDatasetSpecificApi(dataset_specific_api_name, splited_by_frames=splited)
# geting annotations data
print('geting annotations data...')
annotations_data = dataset_specific_api.getAnnotationsData(files)
print('got')
# creating objects iterator
print('creating objects iterator...')
objects = iter(Objects(files, dataset_specific_api.objects_extensions))