def post_process(self, out_folder_base): output_parent_folder = out_folder_base tools.check_dir(output_parent_folder) output_folder = os.path.join( output_parent_folder, self.__class__.__name__) tools.check_dir(output_folder) # image enhancement data = copy.deepcopy(self.output_data) min_value = data.min() max_value = data.max() ratio = 255 / (max_value - min_value) for i in range(self.param['height']): for j in range(self.param['width']): data[i][j] = (data[i][j] - min_value) * ratio # write image out_filename = os.path.join( output_folder, self.metadata[0] + '.png') image.save_img(data, fname=out_filename) log_info = '[IrConverter] save image to: {}'.format(out_filename) print(log_info)
def post_process(self): if self.svfolder_path: if not os.path.isdir(self.svfolder_path): tools.check_dir(self.svfolder_path) elif self.lab_flag: while (self.query_flag): print('\nThere already exists a folder on ' 'save folder path...') ans = input("choosing action\n" "1:Don't save, 2:overwrite? ") if ans == '1': print('Exit the post_process...') self.save_flag = False self.query_flag = False break elif ans == '2': shutil.rmtree(self.svfolder_path) tools.check_dir(self.svfolder_path) self.query_flag = False break else: self.save_flag = True if self.save_flag: basename = os.path.basename(self.img_path) json_name = tools.replace_extension(basename, 'json') o_json_path = os.path.join(self.svfolder_path, json_name) data.write_json(self.results, o_json_path) restore_ordered_json(o_json_path) else: print('Please set saving folder first !') sys.exit(0)
def main_process(self): """ main_process of CreateSymbolicLinkTask """ if "output_folder" in self.param.keys(): tmpdir = self.param["output_folder"] else: tmpdir = tempfile.mkdtemp() self.results = tmpdir tools.check_dir(tmpdir) self.logger.warning("Training data is linked to %s" % tmpdir) image_paths = self.input_data[0] labels = self.input_data[1] if len(image_paths) != len(labels): self.terminate_flag = True self.logger.error("Lengths of labels and data do not match") return False for label in set(labels): if not isinstance(label, str): label = str(label) label_folder = os.path.join(tmpdir, label) tools.check_dir(label_folder) for i in range(0, len(image_paths)): label = labels[i] if not isinstance(label, str): label = str(label) input_path = image_paths[i] fname = os.path.basename(input_path) output_path = os.path.join(tmpdir, label, fname) os.symlink(input_path, output_path) return True
def convert_all_to_png(folder, size=None, keyword=None, outd=None, suffix='png'): """Convert the image to png @param folder: image folder to look for Keyword arguments: suffix -- suffix of the image to be converted size --size of the output image keyword -- keyword of the image files to be converted outd -- output folder for the converted file """ image_paths = find_images(dir_path=folder, keyword=keyword) fail_paths = [] if outd is None: outd = folder else: tools.check_dir(outd) for img_path in image_paths: img = read_img(img_path, log=False) if img is None or img.shape[0] == 0: print('dyda_utils: Error: %s cannot be read' % img_path) continue _fname = os.path.basename(img_path).split('.') _fname[-1] = suffix fname = '.'.join(_fname) fname = os.path.join(outd, fname) save_img(img, fname=fname)
def cp_weights(self): """ copy weight files to snapshot_folder """ print("Copying weights...") cp_folder = os.path.join(self.snapshot_folder, "weights") tools.check_dir(cp_folder) for fname in self.weight_files: ori_path = os.path.join("/tmp", fname) if tools.check_exist(ori_path, log=False): destination = os.path.join(cp_folder, fname) try: copyfile(ori_path, destination) except BaseException: self.logger.error("Fail to copy %s" % ori_path) else: self.logger.error("%s does not exist, exit dyda." % ori_path) sys.exit(0) # FIXME: if there are more TF classifiers in the future classifier = "ClassifierInceptionv3" if self.param["architecture"].find('mobilenet') >= 0: classifier = "ClassifierMobileNet" if classifier in self.config.keys(): new_config = self.config[classifier] new_config["model_file"] = os.path.join(cp_folder, "output_graph.pb") new_config["label_file"] = os.path.join(cp_folder, "output_labels.txt") new_config_path = os.path.join(cp_folder, "dyda.config.learner") tools.write_json({classifier: new_config}, new_config_path)
def post_process(self): if self.sv_flag: tools.check_dir(self.sv_folder) for data in self.output_data: sv_path = os.path.join(self.sv_folder, data["filename"]) sv_path = tools.replace_extension(sv_path, '.json') tools.write_json(data, sv_path) dict_tools.restore_ordered_json(sv_path)
def __init__(self, pipeline_config_path, dyda_config_path="", force_run_skip=False, parent_result_folder="", verbosity=-1, lab_flag=False): """ __init__ of TrainerBase """ self.logger = logging.getLogger('pipeline') formatter1 = logging.Formatter( '[pipeline] %(levelname)s %(message)s' ) log_level = logging.WARNING if verbosity == 1: log_level = logging.INFO elif verbosity >= 2: log_level = logging.DEBUG console1 = logging.StreamHandler() console1.setFormatter(formatter1) self.logger.setLevel(log_level) self.logger.addHandler(console1) self.log_level = log_level self.force_run_skip = force_run_skip self.lab_flag = lab_flag if parent_result_folder == "": cwd = os.getcwd() parent_result_folder = os.path.join(cwd, 'results') self.logger.info('Saving output to %s' % parent_result_folder) tools.check_dir(parent_result_folder) self.parent_result_folder = parent_result_folder self.pipeline_config = tools.parse_json(pipeline_config_path, 'utf-8') if tools.check_exist(dyda_config_path): self.dyda_cfg_path = dyda_config_path elif "dyda_config" in self.pipeline_config.keys(): self.dyda_cfg_path = self.pipeline_config["dyda_config"] else: self.logger.warning( "No valid dyda config found by Pipeline, use default." ) self.dyda_cfg_path = "" self.pipeline = OrderedDict({}) self.define_pipeline() # Output is only set if output_type is specified by a component # If both components set output_type, the later one wins self.output = None self.trigger_level = "L3" if "trigger_level" in self.pipeline_config.keys(): if self.pipeline_config["trigger_level"] in ["L1", "L2", "L3"]: self.trigger_level = self.pipeline_config["trigger_level"] self.logger.info( 'Changing trigger level to %s' % self.trigger_level )
def check_snapshot_folder(self): """ Check if self.snapshot_folder is properly set """ if self.snapshot_folder == "": self.snapshot_folder = os.getcwd() self.snapshot_folder = os.path.join( self.snapshot_folder, self.class_name ) self.base_logger.info( "Creating snapshot folder at %s." % self.snapshot_folder ) tools.check_dir(self.snapshot_folder)
def post_process(self): output_parent_folder = self.lab_output_folder tools.check_dir(output_parent_folder) output_folder = os.path.join(output_parent_folder, self.__class__.__name__) tools.check_dir(output_folder) if len(self.result_list) > 0: for i in range(len(self.result_list)): result = self.result_list[i] out_filename = os.path.join( output_folder, tools.replace_extension(result['filename'], 'json')) data.write_json(result, out_filename)
def main_process(self): """ Main function of dyda component. """ # all input_data should come from the same source and # with the same height and width tools.check_dir(self.snapshot_folder) self.out_path = os.path.join(self.snapshot_folder, self.filename) height, width, layers = self.input_data[0].shape size = (width, height) out = cv2.VideoWriter(self.out_path, cv2.VideoWriter_fourcc(*'DIVX'), 15, size) for i in range(len(self.input_data)): out.write(self.input_data[i]) out.release() self.results = {"output_file": self.out_path}
def post_process(self): if self.svfolder_path: if not os.path.isdir(self.svfolder_path): tools.check_dir(self.svfolder_path) if self.save_flag: i = 0 for out_meta in self.results['additional_info']: name = out_meta['filename'] out_meta['folder'] = os.path.abspath(self.svfolder_path) json_name = tools.replace_extension(name, 'json') json_path = os.path.join(self.svfolder_path, json_name) img_path = os.path.join(self.svfolder_path, name) data.write_json(out_meta, json_path) image.save_img(self.output_data[i], img_path) i += 1 else: print('Please set saving folder first !') sys.exit(0)
def __init__(self, dyda_config_path=''): """ __init__ of LearnerTFClassifier """ super(LearnerYOLO, self).__init__(dyda_config_path=dyda_config_path) self.set_param(self.class_name) self.classes = self.param["labels"] self.overwrite = True if "overwrite" in self.param.keys(): self.overwrite = self.param["overwrite"] if "output_folder" in self.param.keys(): tmpdir = self.param["output_folder"] else: tmpdir = tempfile.mkdtemp() self.output_file = self.param["output_path"] self.label_folder = os.path.join(tmpdir, "labels") self.img_folder = os.path.join(tmpdir, "JPEGImages") for folder in [tmpdir, self.label_folder, self.img_folder]: tools.check_dir(folder) self.darknet_path = self.param["darknet_path"]
def post_process(self, out_folder_base): output_parent_folder = out_folder_base tools.check_dir(output_parent_folder) output_folder = os.path.join( output_parent_folder, self.__class__.__name__) tools.check_dir(output_folder) if 'filename' in self.results.keys(): # write json if self.param['folder'] == "": self.results['folder'] = os.path.join( output_parent_folder, 'IrConverter') out_filename = os.path.join( output_folder, tools.replace_extension(self.results['filename'], 'json')) tools.write_json(self.results, out_filename) log_info = '[TimeScaleShiftConverter] save json to: {}'.format( out_filename) print(log_info)
def read_and_random_crop(fimg, size=None, ratio=0.7, save=False): """Read images and do random crops @param fimg: input image file name Keyword arguments: size -- tuple of new size (default None) ratio -- used to determin the croped size (default 0.7) @return imgs: dictionary of the croped images """ img = read_img(fimg) if img is None: return img nrow = len(img) ncol = len(img[0]) imgs = {} imgs['crop_img_lt'] = img[0:int(nrow * ratio), 0:int(ncol * ratio)] imgs['crop_img_lb'] = img[int(nrow * (1 - ratio)):nrow, 0:int(ncol * ratio)] imgs['crop_img_rt'] = img[0:int(nrow * ratio), int(ncol * (1 - ratio)):ncol] imgs['crop_img_rb'] = img[int(nrow * (1 - ratio)):nrow, int(ncol * (1 - ratio)):ncol] for corner in imgs: if size is not None: imgs[corner] = resize_img(imgs[corner], size) if save: dirname = os.path.dirname(fimg) dirname = os.path.join(dirname, 'crops') tools.check_dir(dirname) _fname = os.path.basename(fimg).split('.') _fname.insert(-1, '_' + corner + '.') fname = ''.join(_fname) fname = os.path.join(dirname, fname) save_img(imgs[corner], fname=fname) return imgs
def snapshot_metadata(self, out_name=''): """ snapshot_metadata will save a copy of the current TrainerBase.metadata as json output :param str out_name: preffix of the output files (default: snapshot) :return: None :rtype: None """ self.base_logger.debug("Creating snapshot of meta for %s" % self.class_name) self.check_snapshot_folder() if self.snapshot_hierarchy: out_folder = self.create_hierarchy_outf('metadata') else: out_folder = os.path.join(self.snapshot_folder, 'metadata') if out_name == "": if isinstance(self.metadata[0], str): out_name = self.metadata[0] else: out_name = "snapshot" try: tools.check_dir(out_folder) except Exception: self.base_logger.error('Cannot create folder %s' % out_folder) traceback.print_exc(file=sys.stdout) try: metadata_path = os.path.join(out_folder, out_name + '.json') data.write_json(self.metadata, fname=metadata_path) self.snapshot_fnames.append(metadata_path) except Exception: self.base_logger.error('Cannot make snapshot for metadata.') traceback.print_exc(file=sys.stdout)
def run(self, external_data, base_name=None, external_meta={}, benchmark=False, force_snapshot=False): """ Main function to be called by external code @param external_data: data passed by the external function, used by component with data_type == "external" """ if benchmark: t0 = time.time() # create two strings to compute time of create DataFrame t_benchmark_start = [] t_benchmark_end = [] t_benchmark_start.append(time.time()) # create pandas DataFrame to store benchmark pd = importlib.import_module('pandas') dfcols = ['total time', 'initial setting', 'run', 'output_handling'] benchmark_data = pd.DataFrame(columns=dfcols) t_benchmark_end.append(time.time()) if base_name is not None: pass else: timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S%f") base_name = "snapshot_" + timestamp logging.info("Using %s as base_name") idx = 0 previous_comp = '' skip_this = False skip_following = False for name, comp in self.pipeline.items(): if benchmark: t1 = time.time() self.logger.debug('running component: %s' % name) # if force_run_skip is on, do not skip if comp['type'] == 'skip': if self.force_run_skip: self.logger.info( 'force_run_skip is on, skip components will be run' ) else: continue comp['instance'].reset() # if it is the first component if idx == 0: self.logger.debug('Set base_name as %s' % base_name) comp['instance'].metadata[0] = base_name comp['instance'].external_metadata = external_meta comp['instance'].external_data = external_data else: comp['instance'].metadata = \ self.pipeline[previous_comp]['instance'].metadata comp['instance'].external_metadata = external_meta comp['instance'].external_data = external_data pipeline_status = 0 if skip_following else 1 comp['instance'].pipeline_status = pipeline_status idx = idx + 1 if skip_this and comp['type'] != 'output_generator': self.logger.debug('Skipping the component %s' % name) comp['instance'].append_results() skip_this = False continue if skip_following and comp['type'] != 'output_generator': self.logger.debug('Skipping the component %s' % name) comp['instance'].append_results() continue input_type = comp['input_type'] self.logger.debug('Setting up input, type: %s' % input_type) if input_type == "use_external_data": comp['instance'].input_data = external_data elif input_type == "package_external_data": comp['instance'].input_data = [external_data] elif input_type == "use_external_meta": comp['instance'].input_data = external_meta elif input_type == "package_external_meta": comp['instance'].input_data = [external_meta] elif input_type == "append_previous_output": comp['instance'].input_data.append( self.pipeline[previous_comp]['instance'].output_data ) elif input_type == "use_metadata": comp['instance'].input_data = \ copy.deepcopy(comp['instance'].metadata) elif input_type == "use_previous_attr": attr = comp['additional_info']['attribute'] if not isinstance(attr, str): self.logger.error( "input_data should be string which match the " "attribute you want to use from previous component" ) sys.exit(0) comp['instance'].input_data = \ getattr(self.pipeline[previous_comp]['instance'], attr) elif input_type == "use_previous_output": comp['instance'].input_data = \ self.pipeline[previous_comp]['instance'].output_data elif input_type == "package_previous_output": comp['instance'].input_data = \ [self.pipeline[previous_comp]['instance'].output_data] elif input_type == "append_previous_results": comp['instance'].input_data.append( self.pipeline[previous_comp]['instance'].results ) elif input_type == "package_previous_results": comp['instance'].input_data = \ [self.pipeline[previous_comp]['instance'].results] elif input_type == "use_previous_results": comp['instance'].input_data = \ self.pipeline[previous_comp]['instance'].results elif input_type == "use": comp['instance'].input_data = external_data if len(comp['additional_info']['input_data']) > 1: self.logger.error( "Input type use can only accept one input, " "please check your pipeline config." ) sys.exit(0) to_use = comp['additional_info']['input_data'][0] if to_use[0] == 'external': if to_use[1] == 'data': comp['instance'].input_data = external_data else: comp['instance'].input_data = external_meta elif to_use[1] == 'results' or to_use[1] == 'metadata': comp['instance'].input_data = \ self.pipeline[to_use[0]]['instance'].results elif to_use[1] == 'package_results': comp['instance'].input_data = \ [self.pipeline[to_use[0]]['instance'].results] elif to_use[1] == 'output_data': comp['instance'].input_data = \ self.pipeline[to_use[0]]['instance'].output_data else: self.logger.error( '%s %s to_use type is not supported' % (to_use[0], to_use[1]) ) elif input_type == "append": if 'additional_info' not in comp.keys(): self.logger.error('No additional_info found') sys.exit(0) if 'input_data' not in comp['additional_info'].keys(): self.logger.error('No input_data session' 'found in additional_info') sys.exit(0) for to_append in comp['additional_info']['input_data']: if to_append[0] == 'external': if to_append[1] == 'data': comp['instance'].input_data.append(external_data) else: comp['instance'].input_data.append(external_meta) elif to_append[1] == 'metadata': comp['instance'].append_metadata_to_input( self.pipeline[to_append[0]]['order'], self.pipeline[to_append[0]]['class'] ) elif to_append[1] == 'results': comp['instance'].input_data.append( self.pipeline[to_append[0]]['instance'].results ) elif to_append[1] == 'package_results': comp['instance'].input_data.append( [self.pipeline[to_append[0]]['instance'].results] ) elif to_append[1] == 'output_data': comp['instance'].input_data.append( self.pipeline[to_append[0]]['instance'].output_data ) else: self.logger.error( '%s %s to_append type is not supported' % (to_append[0], to_append[1]) ) elif input_type == "extend": if 'additional_info' not in comp.keys(): self.logger.error('No additional_info found') sys.exit(0) if 'input_data' not in comp['additional_info'].keys(): self.logger.error('No input_data session' 'found in additional_info') sys.exit(0) for to_extend in comp['additional_info']['input_data']: if to_extend[0] == 'external': if to_extend[1] == 'data': comp['instance'].input_data.extend(external_data) else: comp['instance'].input_data.extend(external_meta) elif to_extend[1] == 'metadata': comp['instance'].extend_input_with_meta( self.pipeline[to_extend[0]]['order'], self.pipeline[to_extend[0]]['class'] ) elif to_extend[1] == 'package_results': comp['instance'].input_data.extend( [self.pipeline[to_append[0]]['instance'].results] ) elif to_extend[1] == 'results': comp['instance'].input_data.extend( self.pipeline[to_extend[0]]['instance'].results ) elif to_extend[1] == 'output_data': comp['instance'].input_data.extend( self.pipeline[to_extend[0]]['instance'].output_data ) else: self.logger.error( '%s %s to_extend type is not supported' % (to_extend[0], to_extend[1]) ) sys.exit(0) elif input_type == "use_meta_pairs": if 'additional_info' not in comp.keys(): self.logger.error('No additional_info found') comp['instance'].input_data = \ comp['additional_info']['comp_key_pairs'] else: self.logger.error('No valid input_type found.') if benchmark: t2 = time.time() snapshot_folder = os.path.join( self.parent_result_folder, comp['instance'].class_name ) comp['instance'].snapshot_folder = snapshot_folder comp['instance'].run() if comp['instance'].terminate_flag: self.logger.warning( 'Component reports terminate_flag, the following normal ' 'components will be skipped. Pipeline will go straight ' 'to output_generator.' ) self.output = None skip_following = True continue if benchmark: t3 = time.time() # selector behavior: if it is a selector, pass the next one if fail if comp['type'] == 'selector': # if the data does not pass selector, e.g. not a key frame if not comp['instance'].output_data: skip_this = True # gate behavior: if it is a gate, pass all following but output elif comp['type'] == 'gate': # if the data does not pass gate, e.g. not a key frame if not comp['instance'].output_data: skip_following = True if 'base_name' in comp.keys(): if isinstance(comp["base_name"], str): comp['instance'].metadata[0] = comp["base_name"] self.logger.warning( "Changing base_name as %s" % comp["base_name"] ) else: self.logger.error("Fail to change base_name") self.logger.debug('metadata: {}'.format(comp['instance'].metadata)) # make snaoshot if output_type is specified if 'output_type' in comp.keys(): if self.lab_flag is True: self.logger.info('Passing output step because ' 'lab_flag is set as True') pass else: do_snapshot = False ''' snapshot == True => always create snapshot force_snapshotable == True => snapshot when force_snapshot is True ''' if 'snapshot' in comp.keys() and comp['snapshot'] is True: do_snapshot = True elif ('force_snapshotable' in comp.keys() and comp['force_snapshotable'] is True): if force_snapshot: do_snapshot = True if comp['output_type'] == 'metadata': self.output = comp['instance'].metadata if do_snapshot: comp['instance'].snapshot_metadata() self.logger.debug('Snapshot for metadata of %s' % comp['instance'].metadata[0]) elif comp['output_type'] == 'output_data': self.output = comp['instance'].output_data if do_snapshot: if False not in [is_pandas_df(i) for i in self.output]: comp['instance'].snapshot_output_data( dtype='DataFrame' ) else: comp['instance'].snapshot_output_data() self.logger.debug('Making snapshot for %s' % comp['instance'].metadata[0]) elif comp['output_type'] == 'unpack_results': if len(comp['instance'].results) >= 1: comp['instance'].results = \ comp['instance'].results[0] self.output = comp['instance'].results else: self.logger.error( "Unpacking results fail, " "keep component results unchanged" ) if do_snapshot: comp['instance'].snapshot_results() self.logger.debug('Snapshot for results of %s' % comp['instance'].metadata[0]) elif comp['output_type'] == 'results': self.output = comp['instance'].results if do_snapshot: comp['instance'].snapshot_results() self.logger.debug('Snapshot for results of %s' % comp['instance'].metadata[0]) elif comp['output_type'] == 'post_process': self.output = [ comp['instance'].results, comp['instance'].metadata, comp['instance'].output_data, ] self.logger.info('Running post_process under' 'lab_flag = %r' % self.lab_flag) comp['instance'].post_process( out_folder_base=self.parent_result_folder ) else: self.logger.warning('%s output_type is not supported' % comp['output_type']) pass comp['output'] = self.output if 'print_output' in comp.keys() and comp['print_output'] is True: print('[pipeline] printing Pipeline.output of %s (%s)' % (comp['instance'].comp_name, comp['instance'].class_name)) print(self.output) if benchmark: t4 = time.time() print('[pipeline] Total time used for %s is %.5f seconds' % (comp['instance'].class_name, t4 - t1)) print(' --- Time used for initial setting is %.5f seconds' % (t2 - t1)) print(' --- Time used for component run is %.5f seconds' % (t3 - t2)) print(' --- Time used for output handling is %.5f seconds' % (t4 - t3)) t_benchmark_start.append(time.time()) component_benchmark = pd.Series( data=[t4 - t1, t2 - t1, t3 - t2, t4 - t3], index=dfcols) benchmark_data.loc[comp['instance'].class_name, :] = \ component_benchmark t_benchmark_end.append(time.time()) previous_comp = name if benchmark: pipeline_time = ((time.time() - t0) - (sum(t_benchmark_end) - sum(t_benchmark_start))) print('Total time used for pipeline.run() is %.5f seconds' % pipeline_time) benchmark_data.ix['Total time used for pipeline.run()', 0] = \ pipeline_time benchmark_data[:] = benchmark_data[:].apply(pd.to_numeric) benchmark_folder = os.path.join(self.parent_result_folder, 'benchmark') tools.check_dir(benchmark_folder) benchmark_path = os.path.join(benchmark_folder, base_name + '.csv') benchmark_data.to_csv(path_or_buf=benchmark_path, float_format='%.5f') return True
def snapshot_output_data(self, dtype="image", out_name=''): """ snapshot_data will save a copy of the current TrainerBase.data based on the data_type set. :param str dtype: data type of output data :param str out_name: preffix of the output files (default: snapshot) :return: None :rtype: None """ self.base_logger.info('Snaoshot output data type: %s.' % dtype) self.check_snapshot_folder() if self.snapshot_hierarchy: out_folder = self.create_hierarchy_outf('output_data') else: out_folder = os.path.join(self.snapshot_folder, 'output_data') if out_name == "": if isinstance(self.metadata[0], str): out_name = self.metadata[0] else: out_name = "snapshot" try: tools.check_dir(out_folder) except Exception: self.base_logger.error('Cannot create folder' % out_folder) traceback.print_exc(file=sys.stdout) try: counter = 0 if not isinstance(self.output_data, list): output = [self.output_data] else: output = self.output_data for output_data_ in output: if dtype == 'image': _tmp = os.path.join( out_folder, out_name + '_tmp.jpg' ) if self.snapshot_with_counter: output_img_path = os.path.join( out_folder, out_name + '_' + str(counter) + '.jpg' ) else: output_img_path = os.path.join( out_folder, out_name + '.jpg' ) image.save_img(output_data_, fname=_tmp) os.rename(_tmp, output_img_path) self.snapshot_fnames.append(output_img_path) elif dtype == 'DataFrame': _tmp = os.path.join( out_folder, out_name + '_tmp.csv' ) if self.snapshot_with_counter: output_img_path = os.path.join( out_folder, out_name + '_' + str(counter) + '.csv' ) else: output_img_path = os.path.join( out_folder, out_name + '.csv' ) output_data_.to_csv(_tmp) os.rename(_tmp, output_img_path) self.snapshot_fnames.append(output_img_path) else: self.base_logger.error('Type %s is not yet supported' % dtype) counter = counter + 1 except Exception: self.base_logger.error('Fail to snapshot for output_data' 'and metadata') traceback.print_exc(file=sys.stdout)
def snapshot_results(self, out_name=''): """ snapshot_results will save a copy of the current TrainerBase.results as json output :param str out_name: preffix of the output files (default: snapshot) :return: None :rtype: None """ self.base_logger.debug("Creating snapshot of results for %s" % self.class_name) self.check_snapshot_folder() if self.snapshot_hierarchy: out_folder = self.create_hierarchy_outf('results') else: out_folder = os.path.join(self.snapshot_folder, 'results') if out_name == "": if isinstance(self.metadata[0], str): out_name = self.metadata[0] else: out_name = "snapshot" try: tools.check_dir(out_folder) except Exception: self.base_logger.error('Cannot create folder %s' % out_folder) traceback.print_exc(file=sys.stdout) try: if isinstance(self.results, list): # snapshot results in only one json if "snapshot_results_all" in self.param.keys() \ and self.param["snapshot_results_all"] is True: results_path = os.path.join( out_folder, out_name + '.json' ) tools.write_json(self.results, fname=results_path) return True # snapshot list of results as different json for i in range(0, len(self.results)): tmp_path = os.path.join( out_folder, out_name + '_tmp.json' ) if self.snapshot_with_counter: results_path = os.path.join( out_folder, out_name + '_' + str(i) + '.json' ) else: results_path = os.path.join( out_folder, out_name + '.json' ) tools.write_json(self.results[i], fname=tmp_path) os.rename(tmp_path, results_path) else: results_path = os.path.join( out_folder, out_name + '.json' ) tools.write_json(self.results, fname=results_path) except Exception: self.base_logger.error('Cannot make snapshot for metadata.') traceback.print_exc(file=sys.stdout) return True
def main_process(self): """ Main function of dyda component. """ self.reset_output() imgs, lab_dicts = self.input_data if not isinstance(imgs, list): imgs = [imgs] self.package_1 = True else: self.package_1 = False if not isinstance(lab_dicts, list): lab_dicts = [lab_dicts] self.package_2 = True else: self.package_2 = False if self.package_1 != self.package_2: self.logger.warning("Inputs should be both list, or both not list") self.terminate_flag = True if self.package_1 and self.package_2: self.package = True else: self.package = False for img, lab_dict in zip(imgs, lab_dicts): annotations = lab_dict["annotations"] boxes = self.get_bounding_boxes(annotations) # face_encodings always return a list of np.ndarray here encodings = face_encodings(img, known_face_locations=boxes, num_jitters=self.num_jitters) self.output_data.append(encodings) if self.save_encodings: face_name = self.external_data[0].split('/')[-2] # only registeration one face in one frame if len(encodings) != 1: self.logger.warning("detect {} faces in {}/{}".format( len(encodings), face_name, self.metadata[0])) break # automatically use the folder name as the face name # if image's path is like /home/shared/NAME/image1.png # use NAME as the face name self.names.append(face_name) encoding = encodings[0] self.known_encodings.append(encoding) data = {"encodings": self.known_encodings, "names": self.names} if self.external_data[0] == self.last_frame: if self.pickle_path == "": tools.check_dir(self.snapshot_folder) self.pickle_path = self.snapshot_folder + \ '/encodings.pickle' self.logger.info("saving face encodings to {}".format( self.pickle_path)) with open(self.pickle_path, 'wb') as wfp: pickle.dump(data, wfp) self.uniform_output()