def __init__(self, samples_path, debug=False, batch_size=1, sample_process_options=SampleProcessor.Options(), output_sample_types=[], person_id_mode=1, generators_count=2, generators_random_seed=None, **kwargs): super().__init__(samples_path, debug, batch_size) self.sample_process_options = sample_process_options self.output_sample_types = output_sample_types self.person_id_mode = person_id_mode if generators_random_seed is not None and len( generators_random_seed) != generators_count: raise ValueError("len(generators_random_seed) != generators_count") self.generators_random_seed = generators_random_seed samples = SampleLoader.load(SampleType.FACE, self.samples_path, person_id_mode=True) if person_id_mode == 1: new_samples = [] for s in samples: new_samples += s samples = new_samples np.random.shuffle(samples) self.samples_len = len(samples) if self.samples_len == 0: raise ValueError('No training data provided.') if self.debug: self.generators_count = 1 self.generators = [ iter_utils.ThisThreadGenerator(self.batch_func, (0, samples)) ] else: self.generators_count = min(generators_count, self.samples_len) if person_id_mode == 1: self.generators = [ iter_utils.SubprocessGenerator( self.batch_func, (i, samples[i::self.generators_count])) for i in range(self.generators_count) ] else: self.generators = [ iter_utils.SubprocessGenerator(self.batch_func, (i, samples)) for i in range(self.generators_count) ] self.generator_counter = -1
def __init__ (self, samples_path, debug=False, batch_size=1, sample_process_options=SampleProcessor.Options(), output_sample_types=[], person_id_mode=1, **kwargs): super().__init__(samples_path, debug, batch_size) self.sample_process_options = sample_process_options self.output_sample_types = output_sample_types self.person_id_mode = person_id_mode samples_host = SampleHost.mp_host (SampleType.FACE, self.samples_path) samples = samples_host.get_list() self.samples_len = len(samples) if self.samples_len == 0: raise ValueError('No training data provided.') unique_person_names = { sample.person_name for sample in samples } persons_name_idxs = { person_name : [] for person_name in unique_person_names } for i,sample in enumerate(samples): persons_name_idxs[sample.person_name].append (i) indexes2D = [ persons_name_idxs[person_name] for person_name in unique_person_names ] index2d_host = mp_utils.Index2DHost(indexes2D) if self.debug: self.generators_count = 1 self.generators = [iter_utils.ThisThreadGenerator ( self.batch_func, (samples_host.create_cli(), index2d_host.create_cli(),) )] else: self.generators_count = np.clip(multiprocessing.cpu_count(), 2, 4) self.generators = [iter_utils.SubprocessGenerator ( self.batch_func, (samples_host.create_cli(), index2d_host.create_cli(),), start_now=True ) for i in range(self.generators_count) ] self.generator_counter = -1
def __init__ (self, samples_path, debug, batch_size, sort_by_yaw=False, sort_by_yaw_target_samples_path=None, with_close_to_self=False, sample_process_options=SampleProcessor.Options(), output_sample_types=[], add_sample_idx=False, generators_count=2, **kwargs): super().__init__(samples_path, debug, batch_size) self.sample_process_options = sample_process_options self.output_sample_types = output_sample_types self.add_sample_idx = add_sample_idx if sort_by_yaw_target_samples_path is not None: self.sample_type = SampleType.FACE_YAW_SORTED_AS_TARGET elif sort_by_yaw: self.sample_type = SampleType.FACE_YAW_SORTED elif with_close_to_self: self.sample_type = SampleType.FACE_WITH_CLOSE_TO_SELF else: self.sample_type = SampleType.FACE self.samples = SampleLoader.load (self.sample_type, self.samples_path, sort_by_yaw_target_samples_path) if self.debug: self.generators_count = 1 self.generators = [iter_utils.ThisThreadGenerator ( self.batch_func, 0 )] else: self.generators_count = min ( generators_count, len(self.samples) ) self.generators = [iter_utils.SubprocessGenerator ( self.batch_func, i ) for i in range(self.generators_count) ] self.generators_sq = [ multiprocessing.Queue() for _ in range(self.generators_count) ] self.generator_counter = -1
def __init__(self, samples_path, debug, batch_size, temporal_image_count, sample_process_options=SampleProcessor.Options(), output_sample_types=[], generators_count=2, **kwargs): super().__init__(samples_path, debug, batch_size) self.temporal_image_count = temporal_image_count self.sample_process_options = sample_process_options self.output_sample_types = output_sample_types self.samples = SampleLoader.load(SampleType.FACE_TEMPORAL_SORTED, self.samples_path) if self.debug: self.generators_count = 1 self.generators = [ iter_utils.ThisThreadGenerator(self.batch_func, 0) ] else: self.generators_count = min(generators_count, len(self.samples)) self.generators = [ iter_utils.SubprocessGenerator(self.batch_func, i) for i in range(self.generators_count) ] self.generator_counter = -1
def __init__(self, trainingdatatype, training_data_path, target_training_data_path=None, debug=False, batch_size=1, **kwargs): if not isinstance(trainingdatatype, TrainingDataType): raise Exception( 'TrainingDataGeneratorBase() trainingdatatype is not TrainingDataType' ) if training_data_path is None: raise Exception('training_data_path is None') self.training_data_path = Path(training_data_path) self.target_training_data_path = Path( target_training_data_path ) if target_training_data_path is not None else None self.debug = debug self.batch_size = 1 if self.debug else batch_size self.trainingdatatype = trainingdatatype self.data = TrainingDataGeneratorBase.load( trainingdatatype, self.training_data_path, self.target_training_data_path) if self.debug: self.generators = [ iter_utils.ThisThreadGenerator(self.batch_func, self.data) ] else: if len(self.data) > 1: self.generators = [ iter_utils.SubprocessGenerator(self.batch_func, self.data[0::2]), iter_utils.SubprocessGenerator(self.batch_func, self.data[1::2]) ] else: self.generators = [ iter_utils.SubprocessGenerator(self.batch_func, self.data) ] self.generator_counter = -1 self.onInitialize(**kwargs)
def __init__(self, samples_path, debug, batch_size, sort_by_yaw=False, sort_by_yaw_target_samples_path=None, sample_process_options=SampleProcessor.Options(), output_sample_types=[], **kwargs): super().__init__(samples_path, debug, batch_size) self.sample_process_options = sample_process_options self.output_sample_types = output_sample_types if sort_by_yaw_target_samples_path is not None: self.sample_type = SampleType.FACE_YAW_SORTED_AS_TARGET elif sort_by_yaw: self.sample_type = SampleType.FACE_YAW_SORTED else: self.sample_type = SampleType.FACE self.samples = SampleLoader.load(self.sample_type, self.samples_path, sort_by_yaw_target_samples_path) if self.debug: self.generator_samples = [self.samples] self.generators = [ iter_utils.ThisThreadGenerator(self.batch_func, 0) ] else: if len(self.samples) > 1: self.generator_samples = [ self.samples[0::2], self.samples[1::2] ] self.generators = [ iter_utils.SubprocessGenerator(self.batch_func, 0), iter_utils.SubprocessGenerator(self.batch_func, 1) ] else: self.generator_samples = [self.samples] self.generators = [ iter_utils.SubprocessGenerator(self.batch_func, 0) ] self.generator_counter = -1
def __init__(self, samples_path, debug, batch_size, sort_by_yaw=False, sort_by_yaw_target_samples_path=None, random_ct_samples_path=None, sample_process_options=SampleProcessor.Options(), output_sample_types=[], add_sample_idx=False, generators_count=2, generators_random_seed=None, **kwargs): super().__init__(samples_path, debug, batch_size) self.sample_process_options = sample_process_options self.output_sample_types = output_sample_types self.add_sample_idx = add_sample_idx if sort_by_yaw_target_samples_path is not None: self.sample_type = SampleType.FACE_YAW_SORTED_AS_TARGET elif sort_by_yaw: self.sample_type = SampleType.FACE_YAW_SORTED else: self.sample_type = SampleType.FACE if generators_random_seed is not None and len( generators_random_seed) != generators_count: raise ValueError("len(generators_random_seed) != generators_count") self.generators_random_seed = generators_random_seed samples = SampleLoader.load(self.sample_type, self.samples_path, sort_by_yaw_target_samples_path) ct_samples = SampleLoader.load( SampleType.FACE, random_ct_samples_path ) if random_ct_samples_path is not None else None self.random_ct_sample_chance = 100 if self.debug: self.generators_count = 1 self.generators = [ iter_utils.ThisThreadGenerator(self.batch_func, (0, samples, ct_samples)) ] else: self.generators_count = min(generators_count, len(samples)) self.generators = [ iter_utils.SubprocessGenerator( self.batch_func, (i, samples[i::self.generators_count], ct_samples)) for i in range(self.generators_count) ] self.generator_counter = -1
def __init__(self, samples_path, debug=False, batch_size=1, random_ct_samples_path=None, sample_process_options=SampleProcessor.Options(), output_sample_types=[], add_sample_idx=False, **kwargs): super().__init__(samples_path, debug, batch_size) self.sample_process_options = sample_process_options self.output_sample_types = output_sample_types self.add_sample_idx = add_sample_idx samples_host = SampleHost.mp_host(SampleType.FACE, self.samples_path) self.samples_len = len(samples_host) if self.samples_len == 0: raise ValueError('No training data provided.') index_host = mp_utils.IndexHost(self.samples_len) if random_ct_samples_path is not None: ct_samples_host = SampleHost.mp_host(SampleType.FACE, random_ct_samples_path) ct_index_host = mp_utils.IndexHost(len(ct_samples_host)) else: ct_samples_host = None ct_index_host = None if self.debug: self.generators_count = 1 self.generators = [ iter_utils.ThisThreadGenerator( self.batch_func, (samples_host.create_cli(), index_host.create_cli(), ct_samples_host.create_cli() if ct_index_host is not None else None, ct_index_host.create_cli() if ct_index_host is not None else None)) ] else: self.generators_count = np.clip(multiprocessing.cpu_count(), 2, 4) self.generators = [ iter_utils.SubprocessGenerator( self.batch_func, (samples_host.create_cli(), index_host.create_cli(), ct_samples_host.create_cli() if ct_index_host is not None else None, ct_index_host.create_cli() if ct_index_host is not None else None), start_now=True) for i in range(self.generators_count) ] self.generator_counter = -1
def __init__(self, samples_path, debug, batch_size, sort_by_yaw=False, sort_by_yaw_target_samples_path=None, sample_process_options=SampleProcessor.Options(), output_sample_types=[], add_sample_idx=False, add_pitch=False, add_yaw=False, generators_count=2, generators_random_seed=None, **kwargs): super().__init__(samples_path, debug, batch_size) self.sample_process_options = sample_process_options self.output_sample_types = output_sample_types self.add_sample_idx = add_sample_idx self.add_pitch = add_pitch self.add_yaw = add_yaw if sort_by_yaw_target_samples_path is not None: self.sample_type = SampleType.FACE_YAW_SORTED_AS_TARGET elif sort_by_yaw: self.sample_type = SampleType.FACE_YAW_SORTED else: self.sample_type = SampleType.FACE self.samples = SampleLoader.load(self.sample_type, self.samples_path, sort_by_yaw_target_samples_path) if generators_random_seed is not None and len( generators_random_seed) != generators_count: raise ValueError("len(generators_random_seed) != generators_count") self.generators_random_seed = generators_random_seed if self.debug: self.generators_count = 1 self.generators = [ iter_utils.ThisThreadGenerator(self.batch_func, 0) ] else: self.generators_count = min(generators_count, len(self.samples)) self.generators = [ iter_utils.SubprocessGenerator(self.batch_func, i) for i in range(self.generators_count) ] self.generators_sq = [ multiprocessing.Queue() for _ in range(self.generators_count) ] self.generator_counter = -1
def __init__ (self, samples_path, debug, batch_size, temporal_image_count, sample_process_options=SampleProcessor.Options(), output_sample_types=[], **kwargs): super().__init__(samples_path, debug, batch_size) self.temporal_image_count = temporal_image_count self.sample_process_options = sample_process_options self.output_sample_types = output_sample_types self.samples = SampleLoader.load (SampleType.IMAGE, self.samples_path) self.generator_samples = [ self.samples ] self.generators = [iter_utils.ThisThreadGenerator ( self.batch_func, 0 )] if self.debug else \ [iter_utils.SubprocessGenerator ( self.batch_func, 0 )] self.generator_counter = -1
def __init__(self, samples_path, debug, batch_size, resolution, sample_process_options=SampleProcessor.Options(), **kwargs): super().__init__(samples_path, debug, batch_size) self.sample_process_options = sample_process_options self.resolution = resolution self.samples = SampleLoader.load( SampleType.FACE_WITH_NEAREST_AS_TARGET, self.samples_path, self.samples_path) if self.debug: self.generator_samples = [self.samples] self.generators = [ iter_utils.ThisThreadGenerator(self.batch_func, 0) ] else: if len(self.samples) > 1: self.generator_samples = [ self.samples[0::2], self.samples[1::2] ] self.generators = [ iter_utils.SubprocessGenerator(self.batch_func, 0), iter_utils.SubprocessGenerator(self.batch_func, 1) ] else: self.generator_samples = [self.samples] self.generators = [ iter_utils.SubprocessGenerator(self.batch_func, 0) ] self.generator_counter = -1
def __init__(self, modelbase, trainingdatatype, batch_size=1, **kwargs): if not isinstance(trainingdatatype, TrainingDataType): raise Exception( 'TrainingDataGeneratorBase() trainingdatatype is not TrainingDataType' ) self.debug = modelbase.is_debug() self.batch_size = 1 if self.debug else batch_size self.trainingdatatype = trainingdatatype self.data = modelbase.get_training_data(trainingdatatype) if self.debug: self.generator = iter_utils.ThisThreadGenerator(self.batch_func) else: self.generator = iter_utils.SubprocessGenerator(self.batch_func) self.onInitialize(**kwargs)
def __init__(self, samples_path, debug, batch_size, temporal_image_count, sample_process_options=SampleProcessor.Options(), output_sample_types=[], generators_count=2, **kwargs): super().__init__(samples_path, debug, batch_size) self.temporal_image_count = temporal_image_count self.sample_process_options = sample_process_options self.output_sample_types = output_sample_types if self.debug: self.generators_count = 1 else: self.generators_count = generators_count samples = SampleHost.load(SampleType.FACE_TEMPORAL_SORTED, self.samples_path) samples_len = len(samples) if samples_len == 0: raise ValueError('No training data provided.') pickled_samples = pickle.dumps(samples, 4) if self.debug: self.generators = [ iter_utils.ThisThreadGenerator(self.batch_func, (0, pickled_samples)) ] else: self.generators = [ iter_utils.SubprocessGenerator(self.batch_func, (i, pickled_samples)) for i in range(self.generators_count) ] self.generator_counter = -1
def __init__ (self, samples_path, debug=False, batch_size=1, sample_process_options=SampleProcessor.Options(), output_sample_types=[], person_id_mode=1, use_caching=False, generators_count=2, generators_random_seed=None, **kwargs): super().__init__(samples_path, debug, batch_size) self.sample_process_options = sample_process_options self.output_sample_types = output_sample_types self.person_id_mode = person_id_mode if generators_random_seed is not None and len(generators_random_seed) != generators_count: raise ValueError("len(generators_random_seed) != generators_count") self.generators_random_seed = generators_random_seed samples = SampleHost.load (SampleType.FACE, self.samples_path, person_id_mode=True, use_caching=use_caching) samples = copy.copy(samples) for i in range(len(samples)): samples[i] = copy.copy(samples[i]) if person_id_mode==1: #np.random.shuffle(samples) # #new_samples = [] #while len(samples) > 0: # for i in range( len(samples)-1, -1, -1): # sample = samples[i] # # if len(sample) > 0: # new_samples.append(sample.pop(0)) # # if len(sample) == 0: # samples.pop(i) # i -= 1 #samples = new_samples new_samples = [] for s in samples: new_samples += s samples = new_samples np.random.shuffle(samples) self.samples_len = len(samples) if self.samples_len == 0: raise ValueError('No training data provided.') if self.debug: self.generators_count = 1 self.generators = [iter_utils.ThisThreadGenerator ( self.batch_func, (0, samples) )] else: self.generators_count = min ( generators_count, self.samples_len ) if person_id_mode==1: self.generators = [iter_utils.SubprocessGenerator ( self.batch_func, (i, samples[i::self.generators_count]) ) for i in range(self.generators_count) ] else: self.generators = [iter_utils.SubprocessGenerator ( self.batch_func, (i, samples) ) for i in range(self.generators_count) ] self.generator_counter = -1