def read_config():
"""
读取配置文件
"""
config_file = os.path.realpath(
os.path.join(os.path.dirname(__file__), 'config.yml'))
if not os.path.exists(config_file):
raise Exception(
'config.yml not exists,you need first `cp config.sample.yml config.yml`,'
+ ' and modify it by you environment')
with open(config_file, 'rt', encoding='utf-8') as f:
config_obj = yaml.safe_load(f.read())
config_obj = AttrDict(config_obj)
if not config_obj.tmpPath:
raise Exception('config.yml not define tmpPath!')
config_obj.tmpPath = os.path.join(os.path.dirname(__file__),
config_obj.tmpPath)
if not config_obj.archivePath:
raise Exception('config.yml not define archivePath!')
config_obj.archivePath = os.path.join(os.path.dirname(__file__),
config_obj.archivePath)
if not config_obj.tasks:
raise Exception('config.yml tasks not defined!')
return config_obj
def get_file_list(self):
# 先获取本地文件列表
archivePath = pydash.get(self.config, 'archivePath')
if not archivePath:
raise Exception("配置缺少archivePath")
local_dir = archivePath + "/" + self.task.name
if os.path.exists(local_dir):
for _dir in os.listdir(local_dir):
self.file_obj_list.append(AttrDict({
"name":_dir,
"size":os.path.getsize(os.path.join(local_dir, _dir)),
"type":"local",
"path":os.path.join(local_dir, _dir)
}))
if self.oss:
fileList = self.oss.get_file_list(os.path.join(self.config.oss.prefix, self.task.name))
self.file_obj_list.extend(fileList)
print('please choice the following file to restore')
if not len(self.file_obj_list):
print(f'task:{self.task.name} has no data to restore, please re-select the task again')
return 'choice_task'
self.file_obj_list = sorted(self.file_obj_list,key=lambda x:x['name'])
for i, file_obj in enumerate(self.file_obj_list):
# print(
# f' {i}) {file_obj["name"]} {int(file_obj["size"]/1024/1024)}MB ({_local_or_remote})')
print(
f' {i}) {file_obj["name"]} {FileHelper.get_size(file_obj["size"])} ({file_obj["type"]})')
print('-1) return last step')
return 'choice_file'
def _decode_cfg_value(v):
"""Decodes a raw config value (e.g., from a yaml config files or command
line argument) into a Python object.
"""
# Configs parsed from raw yaml will contain dictionary keys that need to be
# converted to AttrDict objects
if isinstance(v, dict):
return AttrDict(v)
# All remaining processing is only applied to strings
if not isinstance(v, str):
return v
# Try to interpret `v` as a:
# string, number, tuple, list, dict, boolean, or None
try:
v = literal_eval(v)
# The following two excepts allow v to pass through when it represents a
# string.
#
# Longer explanation:
# The type of v is always a string (before calling literal_eval), but
# sometimes it *represents* a string and other times a data structure, like
# a list. In the case that v represents a string, what we got back from the
# yaml parser is 'foo' *without quotes* (so, not '"foo"'). literal_eval is
# ok with '"foo"', but will raise a ValueError if given 'foo'. In other
# cases, like paths (v = 'foo/bar' and not v = '"foo/bar"'), literal_eval
# will raise a SyntaxError.
except ValueError:
pass
except SyntaxError:
pass
return v
def _merge_cfg_from_file(cfg_filename):
"""Load a yaml config file and merge it into the global config."""
with open(cfg_filename, 'r') as f:
yaml_cfg = yaml.load(f, Loader=yaml.FullLoader)
if yaml_cfg is not None:
_merge_a_into_b(AttrDict(yaml_cfg), __C)
if __C.EXP_NAME == '<fill-with-filename>':
__C.EXP_NAME = os.path.basename(cfg_filename).replace('.yaml', '')
def _get_local_file(self,_path):
print(_path)
_local_files = [f for f in os.listdir(_path) if f.endswith('.zip')]
ret = []
for f in _local_files:
file_path = os.path.join(_path,f)
ret.append(AttrDict({"name":f,
"size":FileHelper.sizeof_fmt(os.path.getsize(file_path)),
"type":'local',
"path":file_path
}))
return ret
from ast import literal_eval import copy import yaml import numpy as np import os import argparse from util.attr_dict import AttrDict __C = AttrDict() cfg = __C # --------------------------------------------------------------------------- # # general options # --------------------------------------------------------------------------- # __C.train = False __C.seed = 42 __C.EXP_NAME = '<fill-with-filename>' __C.GPUS = '0' __C.n_gpus = 2 __C.nodes = 1 __C.nr = 0 __C.fp16 = False __C.fp16_opt_level = "O1" __C.warmup_steps = 13000 __C.logging_steps = 500 __C.weight_decay = 1e-2 __C.DEBUG = False# when you want to debug the code, can set the flag to True __C.SNAPSHOT_FILE = '/home/xdjf/graph-tensor-propagation/exp_gqa/pytorch_ckpt/%s/%04d.ckpt' __C.VOCAB_QUESTION_FILE = '/home/xdjf/graph-tensor-propagation/exp_gqa/data/vocabulary_gqa.txt' __C.VOCAB_ANSWER_FILE = '/home/xdjf/graph-tensor-propagation/exp_gqa/data/answers_gqa.txt'
def merge_cfg_from_file(cfg_filename):
"""Load a yaml config file and merge it into the global config."""
with open(cfg_filename, 'r') as f:
yaml_cfg = AttrDict(yaml.load(f))
_merge_a_into_b(yaml_cfg, __C)
from ast import literal_eval import copy import yaml import numpy as np from util.attr_dict import AttrDict __C = AttrDict() cfg = __C # --------------------------------------------------------------------------- # # general options # --------------------------------------------------------------------------- # __C.EXP_NAME = '__default__' __C.GPU_ID = 0 __C.GPU_MEM_GROWTH = True __C.VOCAB_QUESTION_FILE = './exp_vqa/data/vocabulary_vqa.txt' __C.VOCAB_LAYOUT_FILE = './exp_vqa/data/vocabulary_layout.txt' __C.VOCAB_ANSWER_FILE = './exp_vqa/data/answers_vqa.txt' __C.IMDB_FILE = './exp_vqa/data/imdb/imdb_%s.npy' __C.USE_FIXED_WORD_EMBED = False __C.FIXED_WORD_EMBED_FILE = '' # --------------------------------------------------------------------------- # # model options # --------------------------------------------------------------------------- # __C.MODEL = AttrDict() __C.MODEL.H_FEAT = 14 __C.MODEL.W_FEAT = 14 __C.MODEL.T_CTRL = 12
from ast import literal_eval import copy import yaml import numpy as np import os import argparse from util.attr_dict import AttrDict __C = AttrDict() cfg = __C # --------------------------------------------------------------------------- # # general options # --------------------------------------------------------------------------- # __C.train = False __C.EXP_NAME = '<fill-with-filename>' __C.GPUS = '0' __C.SNAPSHOT_FILE = './exp_clevr/tfmodel/%s/%04d' __C.VOCAB_QUESTION_FILE = './exp_clevr/data/vocabulary_clevr.txt' __C.VOCAB_ANSWER_FILE = './exp_clevr/data/answers_clevr.txt' __C.IMDB_FILE = './exp_clevr/data/imdb/imdb_%s.npy' __C.IMAGE_DIR = './exp_clevr/clevr_dataset/images' __C.SPATIAL_FEATURE_DIR = './exp_clevr/data/features/spatial' __C.FEAT_TYPE = 'spatial' # 'spatial' only, for now __C.INIT_WRD_EMB_FROM_FILE = False __C.WRD_EMB_INIT_FILE = ''
from ast import literal_eval import copy import yaml import numpy as np import os import argparse from util.attr_dict import AttrDict __C = AttrDict() cfg = __C # --------------------------------------------------------------------------- # # general options # --------------------------------------------------------------------------- # __C.train = False __C.EXP_NAME = '<fill-with-filename>' __C.GPUS = '0' __C.SNAPSHOT_FILE = './exp_gqa/tfmodel/%s/%04d' __C.VOCAB_QUESTION_FILE = './exp_gqa/data/vocabulary_gqa.txt' __C.VOCAB_ANSWER_FILE = './exp_gqa/data/answers_gqa.txt' __C.IMDB_FILE = './exp_gqa/gqa_dataset/questions/%s_questions.json' __C.IMAGE_DIR = './exp_gqa/gqa_dataset/images' __C.SPATIAL_FEATURE_DIR = './exp_gqa/gqa_dataset/spatial' __C.OBJECTS_FEATURE_DIR = './exp_gqa/gqa_dataset/objects' # __C.USE_SPATIAL_FEATURE = False __C.FEAT_TYPE = 'objects' # 'spatial', 'objects' or 'scene_graph' # options for "perfect-sight training with ground-truth names & attrs"
from ast import literal_eval import copy import yaml import numpy as np import os import argparse from util.attr_dict import AttrDict __C = AttrDict() cfg = __C # --------------------------------------------------------------------------- # # general options # --------------------------------------------------------------------------- # __C.EXP_NAME = '<fill-with-filename>' __C.GPU_ID = 0 __C.GPU_MEM_GROWTH = True __C.VOCAB_QUESTION_FILE = './exp_clevr_snmn/data/vocabulary_clevr.txt' __C.VOCAB_LAYOUT_FILE = './exp_clevr_snmn/data/vocabulary_layout.txt' __C.VOCAB_ANSWER_FILE = './exp_clevr_snmn/data/answers_clevr.txt' __C.IMDB_FILE = './exp_clevr_snmn/data/imdb/imdb_%s.npy' __C.USE_FIXED_WORD_EMBED = False __C.FIXED_WORD_EMBED_FILE = '' # --------------------------------------------------------------------------- # # model options # --------------------------------------------------------------------------- # __C.MODEL = AttrDict() __C.MODEL.H_FEAT = 14