def __init__ (self, samples_path, debug, batch_size, temporal_image_count, sample_process_options=SampleProcessor.Options(), output_sample_types=[], generators_count=2, **kwargs):
super().__init__(samples_path, debug, batch_size)
self.temporal_image_count = temporal_image_count
self.sample_process_options = sample_process_options
self.output_sample_types = output_sample_types
self.samples = SampleLoader.load (SampleType.FACE_TEMPORAL_SORTED, self.samples_path)
if self.debug:
self.generators_count = 1
self.generators = [iter_utils.ThisThreadGenerator ( self.batch_func, 0 )]
else:
self.generators_count = min ( generators_count, len(self.samples) )
self.generators = [iter_utils.SubprocessGenerator ( self.batch_func, i ) for i in range(self.generators_count) ]
self.generator_counter = -1
def __init__ (self, samples_path, debug=False, batch_size=1,
random_ct_samples_path=None,
sample_process_options=SampleProcessor.Options(),
output_sample_types=[],
uniform_yaw_distribution=False,
generators_count=4,
raise_on_no_data=True,
**kwargs):
super().__init__(debug, batch_size)
self.initialized = False
self.sample_process_options = sample_process_options
self.output_sample_types = output_sample_types
if self.debug:
self.generators_count = 1
else:
self.generators_count = max(1, generators_count)
samples = SampleLoader.load (SampleType.FACE, samples_path)
self.samples_len = len(samples)
if self.samples_len == 0:
if raise_on_no_data:
raise ValueError('No training data provided.')
else:
return
if uniform_yaw_distribution:
samples_pyr = [ ( idx, sample.get_pitch_yaw_roll() ) for idx, sample in enumerate(samples) ]
grads = 128
#instead of math.pi / 2, using -1.2,+1.2 because actually maximum yaw for 2DFAN landmarks are -1.2+1.2
grads_space = np.linspace (-1.2, 1.2,grads)
yaws_sample_list = [None]*grads
for g in io.progress_bar_generator ( range(grads), "Sort by yaw"):
yaw = grads_space[g]
next_yaw = grads_space[g+1] if g < grads-1 else yaw
yaw_samples = []
for idx, pyr in samples_pyr:
s_yaw = -pyr[1]
if (g == 0 and s_yaw < next_yaw) or \
(g < grads-1 and s_yaw >= yaw and s_yaw < next_yaw) or \
(g == grads-1 and s_yaw >= yaw):
yaw_samples += [ idx ]
if len(yaw_samples) > 0:
yaws_sample_list[g] = yaw_samples
yaws_sample_list = [ y for y in yaws_sample_list if y is not None ]
index_host = mplib.Index2DHost( yaws_sample_list )
else:
index_host = mplib.IndexHost(self.samples_len)
if random_ct_samples_path is not None:
ct_samples = SampleLoader.load (SampleType.FACE, random_ct_samples_path)
ct_index_host = mplib.IndexHost( len(ct_samples) )
else:
ct_samples = None
ct_index_host = None
if self.debug:
self.generators = [ThisThreadGenerator ( self.batch_func, (samples, index_host.create_cli(), ct_samples, ct_index_host.create_cli() if ct_index_host is not None else None) )]
else:
self.generators = [SubprocessGenerator ( self.batch_func, (samples, index_host.create_cli(), ct_samples, ct_index_host.create_cli() if ct_index_host is not None else None), start_now=False ) \
for i in range(self.generators_count) ]
SubprocessGenerator.start_in_parallel( self.generators )
self.generator_counter = -1
self.initialized = True
def __init__(self,
samples_path,
debug=False,
batch_size=1,
sample_process_options=SampleProcessor.Options(),
output_sample_types=[],
person_id_mode=1,
use_caching=False,
generators_count=2,
generators_random_seed=None,
**kwargs):
super().__init__(samples_path, debug, batch_size)
self.sample_process_options = sample_process_options
self.output_sample_types = output_sample_types
self.person_id_mode = person_id_mode
if generators_random_seed is not None and len(
generators_random_seed) != generators_count:
raise ValueError("len(generators_random_seed) != generators_count")
self.generators_random_seed = generators_random_seed
samples = SampleLoader.load(SampleType.FACE,
self.samples_path,
person_id_mode=True,
use_caching=use_caching)
if person_id_mode == 1:
np.random.shuffle(samples)
new_samples = []
while len(samples) > 0:
for i in range(len(samples) - 1, -1, -1):
sample = samples[i]
if len(sample) > 0:
new_samples.append(sample.pop(0))
if len(sample) == 0:
samples.pop(i)
samples = new_samples
#new_samples = []
#for s in samples:
# new_samples += s
#samples = new_samples
#np.random.shuffle(samples)
self.samples_len = len(samples)
if self.samples_len == 0:
raise ValueError('No training data provided.')
if self.debug:
self.generators_count = 1
self.generators = [
iter_utils.ThisThreadGenerator(self.batch_func, (0, samples))
]
else:
self.generators_count = min(generators_count, self.samples_len)
if person_id_mode == 1:
self.generators = [
iter_utils.SubprocessGenerator(
self.batch_func,
(i, samples[i::self.generators_count]))
for i in range(self.generators_count)
]
else:
self.generators = [
iter_utils.SubprocessGenerator(self.batch_func,
(i, samples))
for i in range(self.generators_count)
]
self.generator_counter = -1
def __init__(self,
samples_path,
debug=False,
batch_size=1,
sort_by_yaw=False,
sort_by_yaw_target_samples_path=None,
random_ct_samples_path=None,
sample_process_options=SampleProcessor.Options(),
output_sample_types=[],
person_id_mode=False,
add_sample_idx=False,
generators_count=2,
generators_random_seed=None,
**kwargs):
super().__init__(samples_path, debug, batch_size)
self.sample_process_options = sample_process_options
self.output_sample_types = output_sample_types
self.add_sample_idx = add_sample_idx
self.person_id_mode = person_id_mode
if sort_by_yaw_target_samples_path is not None:
self.sample_type = SampleType.FACE_YAW_SORTED_AS_TARGET
elif sort_by_yaw:
self.sample_type = SampleType.FACE_YAW_SORTED
else:
self.sample_type = SampleType.FACE
if generators_random_seed is not None and len(
generators_random_seed) != generators_count:
raise ValueError("len(generators_random_seed) != generators_count")
self.generators_random_seed = generators_random_seed
samples = SampleLoader.load(self.sample_type,
self.samples_path,
sort_by_yaw_target_samples_path,
person_id_mode=person_id_mode)
np.random.shuffle(samples)
self.samples_len = len(samples)
if self.samples_len == 0:
raise ValueError('No training data provided.')
ct_samples = SampleLoader.load(
SampleType.FACE, random_ct_samples_path
) if random_ct_samples_path is not None else None
self.random_ct_sample_chance = 100
if self.debug:
self.generators_count = 1
self.generators = [
iter_utils.ThisThreadGenerator(self.batch_func,
(0, samples, ct_samples))
]
else:
self.generators_count = min(generators_count, self.samples_len)
self.generators = [
iter_utils.SubprocessGenerator(
self.batch_func,
(i, samples[i::self.generators_count], ct_samples))
for i in range(self.generators_count)
]
self.generator_counter = -1
def __init__(self,
samples_path,
debug=False,
batch_size=1,
random_ct_samples_path=None,
sample_process_options=SampleProcessor.Options(),
output_sample_types=[],
add_sample_idx=False,
generators_count=4,
raise_on_no_data=True,
**kwargs):
super().__init__(debug, batch_size)
self.sample_process_options = sample_process_options
self.output_sample_types = output_sample_types
self.add_sample_idx = add_sample_idx
if self.debug:
self.generators_count = 1
else:
self.generators_count = max(1, generators_count)
samples = SampleLoader.load(SampleType.FACE, samples_path)
self.samples_len = len(samples)
self.initialized = False
if self.samples_len == 0:
if raise_on_no_data:
raise ValueError('No training data provided.')
else:
return
index_host = mplib.IndexHost(self.samples_len)
if random_ct_samples_path is not None:
ct_samples = SampleLoader.load(SampleType.FACE,
random_ct_samples_path)
ct_index_host = mplib.IndexHost(len(ct_samples))
else:
ct_samples = None
ct_index_host = None
pickled_samples = pickle.dumps(samples, 4)
ct_pickled_samples = pickle.dumps(
ct_samples, 4) if ct_samples is not None else None
if self.debug:
self.generators = [
ThisThreadGenerator(
self.batch_func,
(pickled_samples, index_host.create_cli(),
ct_pickled_samples, ct_index_host.create_cli()
if ct_index_host is not None else None))
]
else:
self.generators = [SubprocessGenerator ( self.batch_func, (pickled_samples, index_host.create_cli(), ct_pickled_samples, ct_index_host.create_cli() if ct_index_host is not None else None), start_now=False ) \
for i in range(self.generators_count) ]
SubprocessGenerator.start_in_parallel(self.generators)
self.generator_counter = -1
self.initialized = True
