def __init__(self,
samples_path,
debug=False,
batch_size=1,
sample_process_options=SampleProcessor.Options(),
output_sample_types=[],
person_id_mode=1,
generators_count=2,
generators_random_seed=None,
**kwargs):
super().__init__(samples_path, debug, batch_size)
self.sample_process_options = sample_process_options
self.output_sample_types = output_sample_types
self.person_id_mode = person_id_mode
if generators_random_seed is not None and len(
generators_random_seed) != generators_count:
raise ValueError("len(generators_random_seed) != generators_count")
self.generators_random_seed = generators_random_seed
samples = SampleLoader.load(SampleType.FACE,
self.samples_path,
person_id_mode=True)
if person_id_mode == 1:
new_samples = []
for s in samples:
new_samples += s
samples = new_samples
np.random.shuffle(samples)
self.samples_len = len(samples)
if self.samples_len == 0:
raise ValueError('No training data provided.')
if self.debug:
self.generators_count = 1
self.generators = [
iter_utils.ThisThreadGenerator(self.batch_func, (0, samples))
]
else:
self.generators_count = min(generators_count, self.samples_len)
if person_id_mode == 1:
self.generators = [
iter_utils.SubprocessGenerator(
self.batch_func,
(i, samples[i::self.generators_count]))
for i in range(self.generators_count)
]
else:
self.generators = [
iter_utils.SubprocessGenerator(self.batch_func,
(i, samples))
for i in range(self.generators_count)
]
self.generator_counter = -1
def __init__ (self, samples_path, debug=False, batch_size=1,
sample_process_options=SampleProcessor.Options(),
output_sample_types=[],
person_id_mode=1,
**kwargs):
super().__init__(samples_path, debug, batch_size)
self.sample_process_options = sample_process_options
self.output_sample_types = output_sample_types
self.person_id_mode = person_id_mode
samples_host = SampleHost.mp_host (SampleType.FACE, self.samples_path)
samples = samples_host.get_list()
self.samples_len = len(samples)
if self.samples_len == 0:
raise ValueError('No training data provided.')
unique_person_names = { sample.person_name for sample in samples }
persons_name_idxs = { person_name : [] for person_name in unique_person_names }
for i,sample in enumerate(samples):
persons_name_idxs[sample.person_name].append (i)
indexes2D = [ persons_name_idxs[person_name] for person_name in unique_person_names ]
index2d_host = mp_utils.Index2DHost(indexes2D)
if self.debug:
self.generators_count = 1
self.generators = [iter_utils.ThisThreadGenerator ( self.batch_func, (samples_host.create_cli(), index2d_host.create_cli(),) )]
else:
self.generators_count = np.clip(multiprocessing.cpu_count(), 2, 4)
self.generators = [iter_utils.SubprocessGenerator ( self.batch_func, (samples_host.create_cli(), index2d_host.create_cli(),), start_now=True ) for i in range(self.generators_count) ]
self.generator_counter = -1
def __init__ (self, samples_path, debug, batch_size, sort_by_yaw=False, sort_by_yaw_target_samples_path=None, with_close_to_self=False, sample_process_options=SampleProcessor.Options(), output_sample_types=[], add_sample_idx=False, generators_count=2, **kwargs):
super().__init__(samples_path, debug, batch_size)
self.sample_process_options = sample_process_options
self.output_sample_types = output_sample_types
self.add_sample_idx = add_sample_idx
if sort_by_yaw_target_samples_path is not None:
self.sample_type = SampleType.FACE_YAW_SORTED_AS_TARGET
elif sort_by_yaw:
self.sample_type = SampleType.FACE_YAW_SORTED
elif with_close_to_self:
self.sample_type = SampleType.FACE_WITH_CLOSE_TO_SELF
else:
self.sample_type = SampleType.FACE
self.samples = SampleLoader.load (self.sample_type, self.samples_path, sort_by_yaw_target_samples_path)
if self.debug:
self.generators_count = 1
self.generators = [iter_utils.ThisThreadGenerator ( self.batch_func, 0 )]
else:
self.generators_count = min ( generators_count, len(self.samples) )
self.generators = [iter_utils.SubprocessGenerator ( self.batch_func, i ) for i in range(self.generators_count) ]
self.generators_sq = [ multiprocessing.Queue() for _ in range(self.generators_count) ]
self.generator_counter = -1
def __init__(self,
samples_path,
debug,
batch_size,
temporal_image_count,
sample_process_options=SampleProcessor.Options(),
output_sample_types=[],
generators_count=2,
**kwargs):
super().__init__(samples_path, debug, batch_size)
self.temporal_image_count = temporal_image_count
self.sample_process_options = sample_process_options
self.output_sample_types = output_sample_types
self.samples = SampleLoader.load(SampleType.FACE_TEMPORAL_SORTED,
self.samples_path)
if self.debug:
self.generators_count = 1
self.generators = [
iter_utils.ThisThreadGenerator(self.batch_func, 0)
]
else:
self.generators_count = min(generators_count, len(self.samples))
self.generators = [
iter_utils.SubprocessGenerator(self.batch_func, i)
for i in range(self.generators_count)
]
self.generator_counter = -1
def __init__(self,
trainingdatatype,
training_data_path,
target_training_data_path=None,
debug=False,
batch_size=1,
**kwargs):
if not isinstance(trainingdatatype, TrainingDataType):
raise Exception(
'TrainingDataGeneratorBase() trainingdatatype is not TrainingDataType'
)
if training_data_path is None:
raise Exception('training_data_path is None')
self.training_data_path = Path(training_data_path)
self.target_training_data_path = Path(
target_training_data_path
) if target_training_data_path is not None else None
self.debug = debug
self.batch_size = 1 if self.debug else batch_size
self.trainingdatatype = trainingdatatype
self.data = TrainingDataGeneratorBase.load(
trainingdatatype, self.training_data_path,
self.target_training_data_path)
if self.debug:
self.generators = [
iter_utils.ThisThreadGenerator(self.batch_func, self.data)
]
else:
if len(self.data) > 1:
self.generators = [
iter_utils.SubprocessGenerator(self.batch_func,
self.data[0::2]),
iter_utils.SubprocessGenerator(self.batch_func,
self.data[1::2])
]
else:
self.generators = [
iter_utils.SubprocessGenerator(self.batch_func, self.data)
]
self.generator_counter = -1
self.onInitialize(**kwargs)
def __init__(self,
samples_path,
debug,
batch_size,
sort_by_yaw=False,
sort_by_yaw_target_samples_path=None,
sample_process_options=SampleProcessor.Options(),
output_sample_types=[],
**kwargs):
super().__init__(samples_path, debug, batch_size)
self.sample_process_options = sample_process_options
self.output_sample_types = output_sample_types
if sort_by_yaw_target_samples_path is not None:
self.sample_type = SampleType.FACE_YAW_SORTED_AS_TARGET
elif sort_by_yaw:
self.sample_type = SampleType.FACE_YAW_SORTED
else:
self.sample_type = SampleType.FACE
self.samples = SampleLoader.load(self.sample_type, self.samples_path,
sort_by_yaw_target_samples_path)
if self.debug:
self.generator_samples = [self.samples]
self.generators = [
iter_utils.ThisThreadGenerator(self.batch_func, 0)
]
else:
if len(self.samples) > 1:
self.generator_samples = [
self.samples[0::2], self.samples[1::2]
]
self.generators = [
iter_utils.SubprocessGenerator(self.batch_func, 0),
iter_utils.SubprocessGenerator(self.batch_func, 1)
]
else:
self.generator_samples = [self.samples]
self.generators = [
iter_utils.SubprocessGenerator(self.batch_func, 0)
]
self.generator_counter = -1
def __init__(self,
samples_path,
debug,
batch_size,
sort_by_yaw=False,
sort_by_yaw_target_samples_path=None,
random_ct_samples_path=None,
sample_process_options=SampleProcessor.Options(),
output_sample_types=[],
add_sample_idx=False,
generators_count=2,
generators_random_seed=None,
**kwargs):
super().__init__(samples_path, debug, batch_size)
self.sample_process_options = sample_process_options
self.output_sample_types = output_sample_types
self.add_sample_idx = add_sample_idx
if sort_by_yaw_target_samples_path is not None:
self.sample_type = SampleType.FACE_YAW_SORTED_AS_TARGET
elif sort_by_yaw:
self.sample_type = SampleType.FACE_YAW_SORTED
else:
self.sample_type = SampleType.FACE
if generators_random_seed is not None and len(
generators_random_seed) != generators_count:
raise ValueError("len(generators_random_seed) != generators_count")
self.generators_random_seed = generators_random_seed
samples = SampleLoader.load(self.sample_type, self.samples_path,
sort_by_yaw_target_samples_path)
ct_samples = SampleLoader.load(
SampleType.FACE, random_ct_samples_path
) if random_ct_samples_path is not None else None
self.random_ct_sample_chance = 100
if self.debug:
self.generators_count = 1
self.generators = [
iter_utils.ThisThreadGenerator(self.batch_func,
(0, samples, ct_samples))
]
else:
self.generators_count = min(generators_count, len(samples))
self.generators = [
iter_utils.SubprocessGenerator(
self.batch_func,
(i, samples[i::self.generators_count], ct_samples))
for i in range(self.generators_count)
]
self.generator_counter = -1
def __init__(self,
samples_path,
debug=False,
batch_size=1,
random_ct_samples_path=None,
sample_process_options=SampleProcessor.Options(),
output_sample_types=[],
add_sample_idx=False,
**kwargs):
super().__init__(samples_path, debug, batch_size)
self.sample_process_options = sample_process_options
self.output_sample_types = output_sample_types
self.add_sample_idx = add_sample_idx
samples_host = SampleHost.mp_host(SampleType.FACE, self.samples_path)
self.samples_len = len(samples_host)
if self.samples_len == 0:
raise ValueError('No training data provided.')
index_host = mp_utils.IndexHost(self.samples_len)
if random_ct_samples_path is not None:
ct_samples_host = SampleHost.mp_host(SampleType.FACE,
random_ct_samples_path)
ct_index_host = mp_utils.IndexHost(len(ct_samples_host))
else:
ct_samples_host = None
ct_index_host = None
if self.debug:
self.generators_count = 1
self.generators = [
iter_utils.ThisThreadGenerator(
self.batch_func,
(samples_host.create_cli(), index_host.create_cli(),
ct_samples_host.create_cli() if ct_index_host is not None
else None, ct_index_host.create_cli()
if ct_index_host is not None else None))
]
else:
self.generators_count = np.clip(multiprocessing.cpu_count(), 2, 4)
self.generators = [
iter_utils.SubprocessGenerator(
self.batch_func,
(samples_host.create_cli(), index_host.create_cli(),
ct_samples_host.create_cli() if ct_index_host is not None
else None, ct_index_host.create_cli()
if ct_index_host is not None else None),
start_now=True) for i in range(self.generators_count)
]
self.generator_counter = -1
def __init__(self,
samples_path,
debug,
batch_size,
sort_by_yaw=False,
sort_by_yaw_target_samples_path=None,
sample_process_options=SampleProcessor.Options(),
output_sample_types=[],
add_sample_idx=False,
add_pitch=False,
add_yaw=False,
generators_count=2,
generators_random_seed=None,
**kwargs):
super().__init__(samples_path, debug, batch_size)
self.sample_process_options = sample_process_options
self.output_sample_types = output_sample_types
self.add_sample_idx = add_sample_idx
self.add_pitch = add_pitch
self.add_yaw = add_yaw
if sort_by_yaw_target_samples_path is not None:
self.sample_type = SampleType.FACE_YAW_SORTED_AS_TARGET
elif sort_by_yaw:
self.sample_type = SampleType.FACE_YAW_SORTED
else:
self.sample_type = SampleType.FACE
self.samples = SampleLoader.load(self.sample_type, self.samples_path,
sort_by_yaw_target_samples_path)
if generators_random_seed is not None and len(
generators_random_seed) != generators_count:
raise ValueError("len(generators_random_seed) != generators_count")
self.generators_random_seed = generators_random_seed
if self.debug:
self.generators_count = 1
self.generators = [
iter_utils.ThisThreadGenerator(self.batch_func, 0)
]
else:
self.generators_count = min(generators_count, len(self.samples))
self.generators = [
iter_utils.SubprocessGenerator(self.batch_func, i)
for i in range(self.generators_count)
]
self.generators_sq = [
multiprocessing.Queue() for _ in range(self.generators_count)
]
self.generator_counter = -1
def __init__ (self, samples_path, debug, batch_size, temporal_image_count, sample_process_options=SampleProcessor.Options(), output_sample_types=[], **kwargs):
super().__init__(samples_path, debug, batch_size)
self.temporal_image_count = temporal_image_count
self.sample_process_options = sample_process_options
self.output_sample_types = output_sample_types
self.samples = SampleLoader.load (SampleType.IMAGE, self.samples_path)
self.generator_samples = [ self.samples ]
self.generators = [iter_utils.ThisThreadGenerator ( self.batch_func, 0 )] if self.debug else \
[iter_utils.SubprocessGenerator ( self.batch_func, 0 )]
self.generator_counter = -1
def __init__(self,
samples_path,
debug,
batch_size,
resolution,
sample_process_options=SampleProcessor.Options(),
**kwargs):
super().__init__(samples_path, debug, batch_size)
self.sample_process_options = sample_process_options
self.resolution = resolution
self.samples = SampleLoader.load(
SampleType.FACE_WITH_NEAREST_AS_TARGET, self.samples_path,
self.samples_path)
if self.debug:
self.generator_samples = [self.samples]
self.generators = [
iter_utils.ThisThreadGenerator(self.batch_func, 0)
]
else:
if len(self.samples) > 1:
self.generator_samples = [
self.samples[0::2], self.samples[1::2]
]
self.generators = [
iter_utils.SubprocessGenerator(self.batch_func, 0),
iter_utils.SubprocessGenerator(self.batch_func, 1)
]
else:
self.generator_samples = [self.samples]
self.generators = [
iter_utils.SubprocessGenerator(self.batch_func, 0)
]
self.generator_counter = -1
def __init__(self, modelbase, trainingdatatype, batch_size=1, **kwargs):
if not isinstance(trainingdatatype, TrainingDataType):
raise Exception(
'TrainingDataGeneratorBase() trainingdatatype is not TrainingDataType'
)
self.debug = modelbase.is_debug()
self.batch_size = 1 if self.debug else batch_size
self.trainingdatatype = trainingdatatype
self.data = modelbase.get_training_data(trainingdatatype)
if self.debug:
self.generator = iter_utils.ThisThreadGenerator(self.batch_func)
else:
self.generator = iter_utils.SubprocessGenerator(self.batch_func)
self.onInitialize(**kwargs)
def __init__(self,
samples_path,
debug,
batch_size,
temporal_image_count,
sample_process_options=SampleProcessor.Options(),
output_sample_types=[],
generators_count=2,
**kwargs):
super().__init__(samples_path, debug, batch_size)
self.temporal_image_count = temporal_image_count
self.sample_process_options = sample_process_options
self.output_sample_types = output_sample_types
if self.debug:
self.generators_count = 1
else:
self.generators_count = generators_count
samples = SampleHost.load(SampleType.FACE_TEMPORAL_SORTED,
self.samples_path)
samples_len = len(samples)
if samples_len == 0:
raise ValueError('No training data provided.')
pickled_samples = pickle.dumps(samples, 4)
if self.debug:
self.generators = [
iter_utils.ThisThreadGenerator(self.batch_func,
(0, pickled_samples))
]
else:
self.generators = [
iter_utils.SubprocessGenerator(self.batch_func,
(i, pickled_samples))
for i in range(self.generators_count)
]
self.generator_counter = -1
def __init__ (self, samples_path, debug=False, batch_size=1,
sample_process_options=SampleProcessor.Options(),
output_sample_types=[],
person_id_mode=1,
use_caching=False,
generators_count=2,
generators_random_seed=None,
**kwargs):
super().__init__(samples_path, debug, batch_size)
self.sample_process_options = sample_process_options
self.output_sample_types = output_sample_types
self.person_id_mode = person_id_mode
if generators_random_seed is not None and len(generators_random_seed) != generators_count:
raise ValueError("len(generators_random_seed) != generators_count")
self.generators_random_seed = generators_random_seed
samples = SampleHost.load (SampleType.FACE, self.samples_path, person_id_mode=True, use_caching=use_caching)
samples = copy.copy(samples)
for i in range(len(samples)):
samples[i] = copy.copy(samples[i])
if person_id_mode==1:
#np.random.shuffle(samples)
#
#new_samples = []
#while len(samples) > 0:
# for i in range( len(samples)-1, -1, -1):
# sample = samples[i]
#
# if len(sample) > 0:
# new_samples.append(sample.pop(0))
#
# if len(sample) == 0:
# samples.pop(i)
# i -= 1
#samples = new_samples
new_samples = []
for s in samples:
new_samples += s
samples = new_samples
np.random.shuffle(samples)
self.samples_len = len(samples)
if self.samples_len == 0:
raise ValueError('No training data provided.')
if self.debug:
self.generators_count = 1
self.generators = [iter_utils.ThisThreadGenerator ( self.batch_func, (0, samples) )]
else:
self.generators_count = min ( generators_count, self.samples_len )
if person_id_mode==1:
self.generators = [iter_utils.SubprocessGenerator ( self.batch_func, (i, samples[i::self.generators_count]) ) for i in range(self.generators_count) ]
else:
self.generators = [iter_utils.SubprocessGenerator ( self.batch_func, (i, samples) ) for i in range(self.generators_count) ]
self.generator_counter = -1
