def step(self, output_dir): solutions = self.get_current_hparams() params_cudas_dirs = [] solution_results = [] cnt = 0 solutions_modeldirs = {} mkdir(output_dir) for idx, solution in enumerate(solutions): cuda = self.is_cuda_free["free"][0] modeldir = output_dir + "/model-" + str(idx) + "/" log_file = output_dir + "/log-" + str(idx) + ".info" params_cudas_dirs.append([solution, cuda, modeldir, log_file]) solutions_modeldirs[tuple(solution)] = modeldir self.is_cuda_free["free"].remove(cuda) self.is_cuda_free["busy"].append(cuda) if len(params_cudas_dirs ) == self.thread or idx == len(solutions) - 1: tp = ThreadPool(len(params_cudas_dirs)) solution_results += tp.map(self.evaluator.run, params_cudas_dirs) cnt += 1 tp.close() tp.join() for param_cuda in params_cudas_dirs: self.is_cuda_free["free"].append(param_cuda[1]) self.is_cuda_free["busy"].remove(param_cuda[1]) params_cudas_dirs = [] self.feedback(solutions, solution_results) return solutions_modeldirs
def __init__(self, module_home=None): self.local_modules_dir = module_home if module_home else MODULE_HOME self.modules_dict = {} if not os.path.exists(self.local_modules_dir): utils.mkdir(self.local_modules_dir) elif os.path.isfile(self.local_modules_dir): raise ValueError("Module home should be a folder, not a file")
def tb_writer(self): if not os.path.exists(self.config.checkpoint_dir): mkdir(self.config.checkpoint_dir) tb_log_dir = os.path.join(self.config.checkpoint_dir, "visualization") if not self._tb_writer: self._tb_writer = SummaryWriter(tb_log_dir) return self._tb_writer
def report_final_result(result): trial_id = os.environ.get("PaddleHub_AutoDL_Trial_ID") # tmp.txt is to record the eval results for trials mkdir(TMP_HOME) tmp_file = os.path.join(TMP_HOME, "tmp.txt") with open(tmp_file, 'a') as file: file.write(trial_id + "\t" + str(float(result)) + "\n")
def request(self): if not os.path.exists(CACHE_HOME): utils.mkdir(CACHE_HOME) try: r = requests.get(self.get_server_url() + '/' + 'search') data = json.loads(r.text) cache_path = os.path.join(CACHE_HOME, RESOURCE_LIST_FILE) with open(cache_path, 'w+') as fp: yaml.safe_dump({'resource_list': data['data']}, fp) return True except: if self.config.get('debug', False): raise else: pass try: file_url = self.config[ 'resource_storage_server_url'] + RESOURCE_LIST_FILE result, tips, self.resource_list_file = default_downloader.download_file( file_url, save_path=CACHE_HOME, replace=True) if not result: return False except: return False return True
def __init__(self, module_home=None): self.local_modules_dir = module_home if module_home else MODULE_HOME self.modules_dict = {} if not os.path.exists(self.local_modules_dir): utils.mkdir(self.local_modules_dir) elif os.path.isfile(self.local_modules_dir): #TODO(wuzewu): give wanring pass
def vdl_writer(self): """ get vdl_writer for visualization. """ if not os.path.exists(self.config.checkpoint_dir): mkdir(self.config.checkpoint_dir) tb_log_dir = os.path.join(self.config.checkpoint_dir, "visualization") if not self._vdl_writer: self._vdl_writer = LogWriter(tb_log_dir) return self._vdl_writer
def step(self, output_dir): solutions = self.get_current_hparams() params_cudas_dirs = [] solution_results = [] cnt = 0 solutions_modeldirs = {} mkdir(output_dir) solutions = self.mpi.bcast(solutions) # split solutions to "solutions for me" range_start, range_end = self.mpi.split_range(len(solutions)) my_solutions = solutions[range_start:range_end] for idx, solution in enumerate(my_solutions): cuda = self.is_cuda_free["free"][0] modeldir = output_dir + "/model-" + str(idx) + "/" log_file = output_dir + "/log-" + str(idx) + ".info" params_cudas_dirs.append([solution, cuda, modeldir, log_file]) solutions_modeldirs[tuple(solution)] = (modeldir, self.mpi.rank) self.is_cuda_free["free"].remove(cuda) self.is_cuda_free["busy"].append(cuda) if len(params_cudas_dirs ) == self.thread or idx == len(my_solutions) - 1: tp = ThreadPool(len(params_cudas_dirs)) solution_results += tp.map(self.evaluator.run, params_cudas_dirs) cnt += 1 tp.close() tp.join() for param_cuda in params_cudas_dirs: self.is_cuda_free["free"].append(param_cuda[1]) self.is_cuda_free["busy"].remove(param_cuda[1]) params_cudas_dirs = [] all_solution_results = self.mpi.gather(solution_results) if self.mpi.rank == 0: # only rank 0 need to feedback all_solution_results = [y for x in all_solution_results for y in x] self.feedback(solutions, all_solution_results) # remove the tmp.txt which records the eval results for trials tmp_file = os.path.join(TMP_HOME, "tmp.txt") if os.path.exists(tmp_file): os.remove(tmp_file) # collect all solutions_modeldirs collected_solutions_modeldirs = self.mpi.allgather(solutions_modeldirs) return_dict = {} for i in collected_solutions_modeldirs: return_dict.update(i) return return_dict
def download_file(self, url, save_path, save_name=None, retry_limit=3, print_progress=False, replace=False): if not os.path.exists(save_path): utils.mkdir(save_path) save_name = url.split('/')[-1] if save_name is None else save_name file_name = os.path.join(save_path, save_name) retry_times = 0 if replace and os.path.exists(file_name): os.remove(file_name) while not (os.path.exists(file_name)): if os.path.exists(file_name): logger.info("file md5", md5file(file_name)) if retry_times < retry_limit: retry_times += 1 else: tips = "Cannot download {0} within retry limit {1}".format( url, retry_limit) return False, tips, None r = requests.get(url, stream=True) total_length = r.headers.get('content-length') if total_length is None: with open(file_name, 'wb') as f: shutil.copyfileobj(r.raw, f) else: #TODO(ZeyuChen) upgrade to tqdm process with open(file_name, 'wb') as f: dl = 0 total_length = int(total_length) starttime = time.time() if print_progress: print("Downloading %s" % save_name) for data in r.iter_content(chunk_size=4096): dl += len(data) f.write(data) if print_progress: done = int(50 * dl / total_length) progress( "[%-50s] %.2f%%" % ('=' * done, float(dl / total_length * 100))) if print_progress: progress("[%-50s] %.2f%%" % ('=' * 50, 100), end=True) tips = "File %s download completed!" % (file_name) return True, tips, file_name
def _dump_processor(self): import inspect pymodule = inspect.getmodule(self.processor) pycode = inspect.getsource(pymodule) processor_path = self.helper.processor_path() processor_md5 = utils.md5(pycode) processor_md5 += str(time.time()) processor_name = utils.md5(processor_md5) output_file = os.path.join(processor_path, processor_name + ".py") utils.mkdir(processor_path) with open(output_file, "w") as file: file.write(pycode) utils.from_pyobj_to_module_attr( processor_name, self.desc.attr.map.data['processor_info'])
def __init__(self, config_file_path=None): if not config_file_path: config_file_path = os.path.join(hub.CONF_HOME, 'config.json') if not os.path.exists(hub.CONF_HOME): utils.mkdir(hub.CONF_HOME) if not os.path.exists(config_file_path): with open(config_file_path, 'w+') as fp: fp.write(json.dumps(default_server_config)) with open(config_file_path) as fp: self.config = json.load(fp) utils.check_url(self.config['server_url']) self.server_url = self.config['server_url'] self.request() self._load_resource_list_file_if_valid()
def run_cmd(self, argvs): """ Run as a command """ self.parser = argparse.ArgumentParser( description='Run the %s module.' % self.name, prog='hub run %s' % self.name, usage='%(prog)s', add_help=True) self.arg_input_group = self.parser.add_argument_group( title="Input options", description="Input data. Required") self.arg_input_group = self.parser.add_argument_group( title="Ouput options", description="Ouput path. Optional.") self.arg_config_group = self.parser.add_argument_group( title="Config options", description= "Run configuration for controlling module behavior, optional.") self.add_module_config_arg() self.add_module_input_arg() self.add_module_output_arg() args = self.parser.parse_args(argvs) try: input_data = self.check_input_data(args) except DataFormatError and RuntimeError: self.parser.print_help() return None mkdir(args.output_path) wavs, sample_rate = self.synthesize(texts=input_data, use_gpu=args.use_gpu, vocoder=args.vocoder) for index, wav in enumerate(wavs): sf.write(os.path.join(args.output_path, f"{index}.wav"), wav, sample_rate) ret = f"The synthesized wav files have been saved in {args.output_path}" return ret
def __init__(self, config_file_path=None): if not config_file_path: config_file_path = os.path.join(CONF_HOME, 'config.json') if not os.path.exists(CONF_HOME): utils.mkdir(CONF_HOME) if not os.path.exists(config_file_path): with open(config_file_path, 'w+') as fp: lock.flock(fp, lock.LOCK_EX) fp.write(json.dumps(default_server_config)) lock.flock(fp, lock.LOCK_UN) with open(config_file_path, "r") as fp: self.config = json.load(fp) fp_lock = open(config_file_path) lock.flock(fp_lock, lock.LOCK_EX) utils.check_url(self.config['server_url']) self.server_url = self.config['server_url'] self.request() self._load_resource_list_file_if_valid() lock.flock(fp_lock, lock.LOCK_UN)
def __init__(self, feed_list, data_reader, main_program=None, startup_program=None, config=None): # base item self._base_data_reader = data_reader self._base_feed_list = feed_list if main_program is None: self._base_main_program = clone_program( fluid.default_main_program(), for_test=False) else: self._base_main_program = clone_program(main_program, for_test=False) if startup_program is None: self._base_startup_program = clone_program( fluid.default_startup_program(), for_test=False) else: self._base_startup_program = clone_program(startup_program, for_test=False) self.is_checkpoint_loaded = False self._base_compiled_program = None # run config self.config = config if config else RunConfig() self.place = self.places[0] self.device_count = len(self.places) if self.config.use_data_parallel: if not self.config.use_pyreader and self.config.batch_size < self.device_count: logger.warning( "Batch size({}) is less than the count of devices({}), which is not allowed in current Paddle versions" .format(self.config.batch_size, self.device_count)) logger.warning( "Batch size automatically adjusted to {}".format( self.device_count)) self.config._batch_size = self.device_count self.exe = fluid.Executor(place=self.place) self.build_strategy = fluid.BuildStrategy() if self.config.enable_memory_optim: self.build_strategy.memory_optimize = True else: self.build_strategy.memory_optimize = False # log item if not os.path.exists(self.config.checkpoint_dir): mkdir(self.config.checkpoint_dir) vdl_log_dir = os.path.join(self.config.checkpoint_dir, "vdllog") self.log_writer = LogWriter(vdl_log_dir, sync_cycle=1) # run environment self._phases = [] self._envs = {} self._predict_data = None # set default phase self.enter_phase("train")
def __init__(self, feed_list, data_reader, main_program=None, startup_program=None, config=None, metrics_choices="default"): # base item self._base_data_reader = data_reader self._base_feed_list = feed_list # metrics item self.best_score = -999 if metrics_choices == "default": metrics_choices = ["acc"] elif metrics_choices == None: metrics_choices = [] if isinstance(metrics_choices, list): self.metrics_choices = metrics_choices else: self.metrics_choices = [metrics_choices] if main_program is None: self._base_main_program = clone_program( fluid.default_main_program(), for_test=False) else: self._base_main_program = clone_program(main_program, for_test=False) if startup_program is None: self._base_startup_program = clone_program( fluid.default_startup_program(), for_test=False) else: self._base_startup_program = clone_program(startup_program, for_test=False) self.is_checkpoint_loaded = False self._base_compiled_program = None # run config self.config = config if config else RunConfig() self.place = self.places[0] self.device_count = len(self.places) if self.config.use_data_parallel: if not self.config.use_pyreader and self.config.batch_size < self.device_count: logger.warning( "Batch size({}) is less than the count of devices({}), which is not allowed in current Paddle versions" .format(self.config.batch_size, self.device_count)) logger.warning( "Batch size automatically adjusted to {}".format( self.device_count)) self.config._batch_size = self.device_count self.exe = fluid.Executor(place=self.place) self.build_strategy = fluid.BuildStrategy() # log item if not os.path.exists(self.config.checkpoint_dir): mkdir(self.config.checkpoint_dir) tb_log_dir = os.path.join(self.config.checkpoint_dir, "visualization") self.tb_writer = SummaryWriter(tb_log_dir) # run environment self._phases = [] self._envs = {} self._predict_data = None # accelerate predict self.is_best_model_loaded = False # set default phase self.enter_phase("train")
def serialize_to_path(self, path=None, exe=None): self._check_signatures() self._generate_desc() # create module path for saving if path is None: path = os.path.join(".", self.name) self.helper = ModuleHelper(path) utils.mkdir(self.helper.module_dir) # create module pb module_desc = module_desc_pb2.ModuleDesc() logger.info("PaddleHub version = %s" % version.hub_version) logger.info("PaddleHub Module proto version = %s" % version.module_proto_version) logger.info("Paddle version = %s" % paddle.__version__) feeded_var_names = [ input.name for key, sign in self.signatures.items() for input in sign.inputs ] target_vars = [ output for key, sign in self.signatures.items() for output in sign.outputs ] feeded_var_names = list(set(feeded_var_names)) target_vars = list(set(target_vars)) # save inference program program = self.program.clone() for block in program.blocks: for op in block.ops: if "op_callstack" in op.all_attrs(): op._set_attr("op_callstack", [""]) if not exe: place = fluid.CPUPlace() exe = fluid.Executor(place=place) utils.mkdir(self.helper.model_path()) fluid.io.save_inference_model( self.helper.model_path(), feeded_var_names=list(feeded_var_names), target_vars=list(target_vars), main_program=program, executor=exe) with open(os.path.join(self.helper.model_path(), "__model__"), "rb") as file: program_desc_str = file.read() rename_program = fluid.framework.Program.parse_from_string( program_desc_str) varlist = { var: block for block in rename_program.blocks for var in block.vars if self.get_name_prefix() not in var } for var, block in varlist.items(): old_name = var new_name = self.get_var_name_with_prefix(old_name) block._rename_var(old_name, new_name) utils.mkdir(self.helper.model_path()) with open( os.path.join(self.helper.model_path(), "__model__"), "wb") as f: f.write(rename_program.desc.serialize_to_string()) for file in os.listdir(self.helper.model_path()): if (file == "__model__" or self.get_name_prefix() in file): continue os.rename( os.path.join(self.helper.model_path(), file), os.path.join(self.helper.model_path(), self.get_var_name_with_prefix(file))) # create processor file if self.processor: self._dump_processor() # create assets self._dump_assets() # create check info checker = ModuleChecker(self.helper.module_dir) checker.generate_check_info() # Serialize module_desc pb module_pb = self.desc.SerializeToString() with open(self.helper.module_desc_path(), "wb") as f: f.write(module_pb)
def _dump_assets(self): utils.mkdir(self.helper.assets_path()) for asset in self.assets: filename = os.path.basename(asset) newfile = os.path.join(self.helper.assets_path(), filename) copyfile(asset, newfile)