def __init__ (self, samples_path, debug, batch_size, temporal_image_count, sample_process_options=SampleProcessor.Options(), output_sample_types=[], generators_count=2, **kwargs): super().__init__(samples_path, debug, batch_size) self.temporal_image_count = temporal_image_count self.sample_process_options = sample_process_options self.output_sample_types = output_sample_types self.samples = SampleLoader.load (SampleType.FACE_TEMPORAL_SORTED, self.samples_path) if self.debug: self.generators_count = 1 self.generators = [iter_utils.ThisThreadGenerator ( self.batch_func, 0 )] else: self.generators_count = min ( generators_count, len(self.samples) ) self.generators = [iter_utils.SubprocessGenerator ( self.batch_func, i ) for i in range(self.generators_count) ] self.generator_counter = -1
def __init__ (self, samples_path, debug=False, batch_size=1, random_ct_samples_path=None, sample_process_options=SampleProcessor.Options(), output_sample_types=[], uniform_yaw_distribution=False, generators_count=4, raise_on_no_data=True, **kwargs): super().__init__(debug, batch_size) self.initialized = False self.sample_process_options = sample_process_options self.output_sample_types = output_sample_types if self.debug: self.generators_count = 1 else: self.generators_count = max(1, generators_count) samples = SampleLoader.load (SampleType.FACE, samples_path) self.samples_len = len(samples) if self.samples_len == 0: if raise_on_no_data: raise ValueError('No training data provided.') else: return if uniform_yaw_distribution: samples_pyr = [ ( idx, sample.get_pitch_yaw_roll() ) for idx, sample in enumerate(samples) ] grads = 128 #instead of math.pi / 2, using -1.2,+1.2 because actually maximum yaw for 2DFAN landmarks are -1.2+1.2 grads_space = np.linspace (-1.2, 1.2,grads) yaws_sample_list = [None]*grads for g in io.progress_bar_generator ( range(grads), "Sort by yaw"): yaw = grads_space[g] next_yaw = grads_space[g+1] if g < grads-1 else yaw yaw_samples = [] for idx, pyr in samples_pyr: s_yaw = -pyr[1] if (g == 0 and s_yaw < next_yaw) or \ (g < grads-1 and s_yaw >= yaw and s_yaw < next_yaw) or \ (g == grads-1 and s_yaw >= yaw): yaw_samples += [ idx ] if len(yaw_samples) > 0: yaws_sample_list[g] = yaw_samples yaws_sample_list = [ y for y in yaws_sample_list if y is not None ] index_host = mplib.Index2DHost( yaws_sample_list ) else: index_host = mplib.IndexHost(self.samples_len) if random_ct_samples_path is not None: ct_samples = SampleLoader.load (SampleType.FACE, random_ct_samples_path) ct_index_host = mplib.IndexHost( len(ct_samples) ) else: ct_samples = None ct_index_host = None if self.debug: self.generators = [ThisThreadGenerator ( self.batch_func, (samples, index_host.create_cli(), ct_samples, ct_index_host.create_cli() if ct_index_host is not None else None) )] else: self.generators = [SubprocessGenerator ( self.batch_func, (samples, index_host.create_cli(), ct_samples, ct_index_host.create_cli() if ct_index_host is not None else None), start_now=False ) \ for i in range(self.generators_count) ] SubprocessGenerator.start_in_parallel( self.generators ) self.generator_counter = -1 self.initialized = True
def __init__(self, samples_path, debug=False, batch_size=1, sample_process_options=SampleProcessor.Options(), output_sample_types=[], person_id_mode=1, use_caching=False, generators_count=2, generators_random_seed=None, **kwargs): super().__init__(samples_path, debug, batch_size) self.sample_process_options = sample_process_options self.output_sample_types = output_sample_types self.person_id_mode = person_id_mode if generators_random_seed is not None and len( generators_random_seed) != generators_count: raise ValueError("len(generators_random_seed) != generators_count") self.generators_random_seed = generators_random_seed samples = SampleLoader.load(SampleType.FACE, self.samples_path, person_id_mode=True, use_caching=use_caching) if person_id_mode == 1: np.random.shuffle(samples) new_samples = [] while len(samples) > 0: for i in range(len(samples) - 1, -1, -1): sample = samples[i] if len(sample) > 0: new_samples.append(sample.pop(0)) if len(sample) == 0: samples.pop(i) samples = new_samples #new_samples = [] #for s in samples: # new_samples += s #samples = new_samples #np.random.shuffle(samples) self.samples_len = len(samples) if self.samples_len == 0: raise ValueError('No training data provided.') if self.debug: self.generators_count = 1 self.generators = [ iter_utils.ThisThreadGenerator(self.batch_func, (0, samples)) ] else: self.generators_count = min(generators_count, self.samples_len) if person_id_mode == 1: self.generators = [ iter_utils.SubprocessGenerator( self.batch_func, (i, samples[i::self.generators_count])) for i in range(self.generators_count) ] else: self.generators = [ iter_utils.SubprocessGenerator(self.batch_func, (i, samples)) for i in range(self.generators_count) ] self.generator_counter = -1
def __init__(self, samples_path, debug=False, batch_size=1, sort_by_yaw=False, sort_by_yaw_target_samples_path=None, random_ct_samples_path=None, sample_process_options=SampleProcessor.Options(), output_sample_types=[], person_id_mode=False, add_sample_idx=False, generators_count=2, generators_random_seed=None, **kwargs): super().__init__(samples_path, debug, batch_size) self.sample_process_options = sample_process_options self.output_sample_types = output_sample_types self.add_sample_idx = add_sample_idx self.person_id_mode = person_id_mode if sort_by_yaw_target_samples_path is not None: self.sample_type = SampleType.FACE_YAW_SORTED_AS_TARGET elif sort_by_yaw: self.sample_type = SampleType.FACE_YAW_SORTED else: self.sample_type = SampleType.FACE if generators_random_seed is not None and len( generators_random_seed) != generators_count: raise ValueError("len(generators_random_seed) != generators_count") self.generators_random_seed = generators_random_seed samples = SampleLoader.load(self.sample_type, self.samples_path, sort_by_yaw_target_samples_path, person_id_mode=person_id_mode) np.random.shuffle(samples) self.samples_len = len(samples) if self.samples_len == 0: raise ValueError('No training data provided.') ct_samples = SampleLoader.load( SampleType.FACE, random_ct_samples_path ) if random_ct_samples_path is not None else None self.random_ct_sample_chance = 100 if self.debug: self.generators_count = 1 self.generators = [ iter_utils.ThisThreadGenerator(self.batch_func, (0, samples, ct_samples)) ] else: self.generators_count = min(generators_count, self.samples_len) self.generators = [ iter_utils.SubprocessGenerator( self.batch_func, (i, samples[i::self.generators_count], ct_samples)) for i in range(self.generators_count) ] self.generator_counter = -1
def __init__(self, samples_path, debug=False, batch_size=1, random_ct_samples_path=None, sample_process_options=SampleProcessor.Options(), output_sample_types=[], add_sample_idx=False, generators_count=4, raise_on_no_data=True, **kwargs): super().__init__(debug, batch_size) self.sample_process_options = sample_process_options self.output_sample_types = output_sample_types self.add_sample_idx = add_sample_idx if self.debug: self.generators_count = 1 else: self.generators_count = max(1, generators_count) samples = SampleLoader.load(SampleType.FACE, samples_path) self.samples_len = len(samples) self.initialized = False if self.samples_len == 0: if raise_on_no_data: raise ValueError('No training data provided.') else: return index_host = mplib.IndexHost(self.samples_len) if random_ct_samples_path is not None: ct_samples = SampleLoader.load(SampleType.FACE, random_ct_samples_path) ct_index_host = mplib.IndexHost(len(ct_samples)) else: ct_samples = None ct_index_host = None pickled_samples = pickle.dumps(samples, 4) ct_pickled_samples = pickle.dumps( ct_samples, 4) if ct_samples is not None else None if self.debug: self.generators = [ ThisThreadGenerator( self.batch_func, (pickled_samples, index_host.create_cli(), ct_pickled_samples, ct_index_host.create_cli() if ct_index_host is not None else None)) ] else: self.generators = [SubprocessGenerator ( self.batch_func, (pickled_samples, index_host.create_cli(), ct_pickled_samples, ct_index_host.create_cli() if ct_index_host is not None else None), start_now=False ) \ for i in range(self.generators_count) ] SubprocessGenerator.start_in_parallel(self.generators) self.generator_counter = -1 self.initialized = True