def run(self, *args):
params = args[0][0]
num_cuda = args[0][1]
saved_params_dir = args[0][2]
log_file = args[0][3]
params = self.convert_params(params)
if not self.is_valid_params(params):
return REWARD_SUM
param_str = self.format_params_str(params)
f = open(log_file, "w")
f.close()
if len(self.half_best_model_path) > 0:
model_path = self.half_best_model_path[self.run_count % len(
self.half_best_model_path)]
if is_windows():
run_cmd = "set FLAGS_eager_delete_tensor_gb=0.0&set CUDA_VISIBLE_DEVICES=%s&python -u %s --epochs=1 --model_path %s --saved_params_dir=%s %s %s >%s 2>&1" % \
(num_cuda, self.finetunee_script, model_path, saved_params_dir, param_str, self.options_str, log_file)
else:
run_cmd = "export FLAGS_eager_delete_tensor_gb=0.0; export CUDA_VISIBLE_DEVICES=%s; python -u %s --epochs=1 --model_path %s --saved_params_dir=%s %s %s >%s 2>&1" % \
(num_cuda, self.finetunee_script, model_path, saved_params_dir, param_str, self.options_str, log_file)
else:
if is_windows():
run_cmd = "set FLAGS_eager_delete_tensor_gb=0.0&set CUDA_VISIBLE_DEVICES=%s&python -u %s --saved_params_dir=%s %s %s >%s 2>&1" % \
(num_cuda, self.finetunee_script, saved_params_dir, param_str, self.options_str, log_file)
else:
run_cmd = "export FLAGS_eager_delete_tensor_gb=0.0; export CUDA_VISIBLE_DEVICES=%s; python -u %s --saved_params_dir=%s %s %s >%s 2>&1" % \
(num_cuda, self.finetunee_script, saved_params_dir, param_str, self.options_str, log_file)
self.run_count += 1
try:
os.system(run_cmd)
with open(log_file, "r") as f:
lines = f.readlines()
eval_result = []
for line in lines:
line = line.strip()
if line.startswith("AutoFinetuneEval"):
data = line.split("\t")
eval_result = float(data[-1])
if eval_result == []:
print(
"WARNING: Program which was ran with hyperparameters as %s was crashed!"
% param_str.replace("--", ""))
eval_result = 0.0
except:
print(
"WARNING: Program which was ran with hyperparameters as %s was crashed!"
% param_str.replace("--", ""))
eval_result = 0.0
reward = self.get_reward(eval_result)
self.model_rewards[saved_params_dir] = reward
return reward
def execute(self, argv):
if not argv:
argv = ['.*']
resource_name = argv[0]
extra = {"command": "search"}
resource_list = default_hub_server.search_resource(
resource_name, resource_type="Module", extra=extra)
if utils.is_windows():
placeholders = [20, 8, 8, 20]
else:
placeholders = [30, 8, 8, 25]
tp = TablePrinter(
titles=["ResourceName", "Type", "Version", "Summary"],
placeholders=placeholders)
if len(resource_list) == 0:
if default_hub_server._server_check() is False:
print(
"Request Hub-Server unsuccessfully, please check your network."
)
for resource_name, resource_type, resource_version, resource_summary in resource_list:
if resource_type == "Module":
colors = ["yellow", None, None, None]
else:
colors = ["light_red", None, None, None]
tp.add_line(contents=[
resource_name, resource_type, resource_version,
resource_summary
],
colors=colors)
print(tp.get_text())
return True
def execute(self, argv):
if not argv:
argv = ['.*']
resource_name = argv[0]
resource_list = default_hub_server.search_resource(
resource_name, resource_type="Module")
if utils.is_windows():
placeholders = [20, 8, 8, 20]
else:
placeholders = [30, 8, 8, 25]
tp = TablePrinter(
titles=["ResourceName", "Type", "Version", "Summary"],
placeholders=placeholders)
for resource_name, resource_type, resource_version, resource_summary in resource_list:
if resource_type == "Module":
colors = ["yellow", None, None, None]
else:
colors = ["light_red", None, None, None]
tp.add_line(contents=[
resource_name, resource_type, resource_version,
resource_summary
],
colors=colors)
print(tp.get_text())
return True
def show_module_info(self, module_dir):
module = Module(module_dir=module_dir)
if utils.is_windows():
placeholders = [15, 40]
else:
placeholders = [15, 50]
tp = TablePrinter(titles=["ModuleName", module.name],
placeholders=placeholders,
title_colors=["light_red", None],
title_aligns=["^", "<"])
tp.add_line(contents=["Version", module.version],
colors=["light_red", None],
aligns=["^", "<"])
tp.add_line(contents=["Summary", module.summary],
colors=["light_red", None],
aligns=["^", "<"])
tp.add_line(contents=["Author", module.author],
colors=["light_red", None],
aligns=["^", "<"])
tp.add_line(contents=["Author-Email", module.author_email],
colors=["light_red", None],
aligns=["^", "<"])
tp.add_line(contents=["Location", module_dir[0]],
colors=["light_red", None],
aligns=["^", "<"])
print(tp.get_text())
return True
def show_model_info(self, model_info_file):
model_info = yaml_parser.parse(model_info_file)
if utils.is_windows():
placeholders = [15, 40]
else:
placeholders = [15, 50]
tp = TablePrinter(titles=["ModelName", model_info['name']],
placeholders=placeholders,
title_colors=["yellow", None],
title_aligns=["^", "<"])
tp.add_line(contents=["Type", model_info['type']],
colors=["yellow", None],
aligns=["^", "<"])
tp.add_line(contents=["Version", model_info['version']],
colors=["yellow", None],
aligns=["^", "<"])
tp.add_line(contents=["Summary", model_info['description']],
colors=["yellow", None],
aligns=["^", "<"])
tp.add_line(contents=["Author", model_info['author']],
colors=["yellow", None],
aligns=["^", "<"])
tp.add_line(contents=["Author-Email", model_info['author_email']],
colors=["yellow", None],
aligns=["^", "<"])
print(tp.get_text())
return True
def execute(self, argv):
all_modules = default_module_manager.all_modules()
if utils.is_windows():
placeholders = [20, 40]
else:
placeholders = [25, 50]
tp = TablePrinter(
titles=["ModuleName", "Path"], placeholders=placeholders)
for module_name, module_dir in all_modules.items():
tp.add_line(contents=[module_name, module_dir[0]])
print(tp.get_text())
return True
def run(self, *args):
params = args[0][0]
num_cuda = args[0][1]
saved_params_dir = args[0][2]
log_file = args[0][3]
params = self.convert_params(params)
if not self.is_valid_params(params):
return REWARD_SUM
param_str = self.format_params_str(params)
f = open(log_file, "w")
f.close()
if is_windows():
run_cmd = "set FLAGS_eager_delete_tensor_gb=0.0&set CUDA_VISIBLE_DEVICES=%s&python -u %s --saved_params_dir=%s %s %s >%s 2>&1" % \
(num_cuda, self.finetunee_script, saved_params_dir, param_str, self.options_str, log_file)
else:
run_cmd = "export FLAGS_eager_delete_tensor_gb=0.0; export CUDA_VISIBLE_DEVICES=%s; python -u %s --saved_params_dir=%s %s %s >%s 2>&1" % \
(num_cuda, self.finetunee_script, saved_params_dir, param_str, self.options_str, log_file)
try:
# set temp environment variable to record the eval results for trials
rand_str = unique_name()
os.environ['PaddleHub_AutoDL_Trial_ID'] = rand_str
os.system(run_cmd)
eval_result = []
tmp_file = os.path.join(TMP_HOME, 'tmp.txt')
with open(tmp_file, 'r') as file:
for line in file:
data = line.strip().split("\t")
if rand_str == data[0]:
eval_result = float(data[1])
if eval_result == []:
print(
"WARNING: Program which was ran with hyperparameters as %s was crashed!"
% param_str.replace("--", ""))
eval_result = 0.0
except:
print(
"WARNING: Program which was ran with hyperparameters as %s was crashed!"
% param_str.replace("--", ""))
eval_result = 0.0
reward = self.get_reward(eval_result)
self.model_rewards[saved_params_dir] = reward
return reward
def paint_modules_info(module_versions_info):
if is_windows():
placeholders = [20, 8, 14, 14]
else:
placeholders = [30, 8, 16, 16]
tp = TablePrinter(
titles=["ResourceName", "Version", "PaddlePaddle", "PaddleHub"],
placeholders=placeholders)
module_versions_info.sort(key=cmp_to_key(sort_version_key))
for resource_name, resource_version, paddle_version, \
hub_version in module_versions_info:
colors = ["yellow", None, None, None]
tp.add_line(contents=[
resource_name, resource_version,
strflist_version(paddle_version),
strflist_version(hub_version)
],
colors=colors)
return tp.get_text()
def install_module(self,
module_name=None,
module_dir=None,
module_package=None,
module_version=None,
upgrade=False,
extra=None):
md5_value = installed_module_version = None
from_user_dir = True if module_dir else False
with tmp_dir() as _dir:
if module_name:
self.all_modules(update=True)
module_info = self.modules_dict.get(module_name, None)
if module_info:
if not module_version or module_version == self.modules_dict[
module_name][1]:
module_dir = self.modules_dict[module_name][0]
module_tag = module_name if not module_version else '%s-%s' % (
module_name, module_version)
tips = "Module %s already installed in %s" % (
module_tag, module_dir)
return True, tips, self.modules_dict[module_name]
search_result = hub.HubServer().get_module_url(
module_name, version=module_version, extra=extra)
name = search_result.get('name', None)
url = search_result.get('url', None)
md5_value = search_result.get('md5', None)
installed_module_version = search_result.get('version', None)
if not url or (module_version is not None
and installed_module_version != module_version
) or (name != module_name):
if hub.HubServer()._server_check() is False:
tips = "Request Hub-Server unsuccessfully, please check your network."
return False, tips, None
module_versions_info = hub.HubServer().search_module_info(
module_name)
if module_versions_info is not None and len(
module_versions_info) > 0:
if utils.is_windows():
placeholders = [20, 8, 14, 14]
else:
placeholders = [30, 8, 16, 16]
tp = TablePrinter(titles=[
"ResourceName", "Version", "PaddlePaddle",
"PaddleHub"
],
placeholders=placeholders)
module_versions_info.sort(
key=cmp_to_key(utils.sort_version_key))
for resource_name, resource_version, paddle_version, \
hub_version in module_versions_info:
colors = ["yellow", None, None, None]
tp.add_line(contents=[
resource_name, resource_version,
utils.strflist_version(paddle_version),
utils.strflist_version(hub_version)
],
colors=colors)
tips = "The version of PaddlePaddle or PaddleHub " \
"can not match module, please upgrade your " \
"PaddlePaddle or PaddleHub according to the form " \
"below." + tp.get_text()
else:
tips = "Can't find module %s" % module_name
if module_version:
tips += " with version %s" % module_version
return False, tips, None
result, tips, module_zip_file = default_downloader.download_file(
url=url,
save_path=_dir,
save_name=module_name,
replace=True,
print_progress=True)
result, tips, module_dir = default_downloader.uncompress(
file=module_zip_file,
dirname=MODULE_HOME,
delete_file=True,
print_progress=True)
if module_package:
with tarfile.open(module_package, "r:gz") as tar:
file_names = tar.getnames()
size = len(file_names) - 1
module_dir = os.path.split(file_names[0])[0]
module_dir = os.path.join(_dir, module_dir)
for index, file_name in enumerate(file_names):
tar.extract(file_name, _dir)
if module_dir:
if not module_name:
module_name = hub.Module(directory=module_dir).name
self.all_modules(update=False)
module_info = self.modules_dict.get(module_name, None)
if module_info:
module_dir = self.modules_dict[module_name][0]
module_tag = module_name if not module_version else '%s-%s' % (
module_name, module_version)
tips = "Module %s already installed in %s" % (module_tag,
module_dir)
return True, tips, self.modules_dict[module_name]
if module_dir:
if md5_value:
with open(os.path.join(MODULE_HOME, module_dir, "md5.txt"),
"w") as fp:
fp.write(md5_value)
save_path = os.path.join(MODULE_HOME, module_name)
if os.path.exists(save_path):
shutil.move(save_path)
if from_user_dir:
shutil.copytree(module_dir, save_path)
else:
shutil.move(module_dir, save_path)
module_dir = save_path
tips = "Successfully installed %s" % module_name
if installed_module_version:
tips += "-%s" % installed_module_version
return True, tips, (module_dir, installed_module_version)
tips = "Download %s-%s failed" % (module_name, module_version)
return False, tips, module_dir
def get_ph_value():
if is_windows():
return 0
return 11
def get_color_dict():
if is_windows():
return windows_color_dict
return linux_color_dict
