def compare_opts(opt_path_1: str, opt_path_2: str, load_raw: bool = False) -> str:
"""
Super simple script to compare the contents of two .opt files.
Return the formatted text comparing the two .opts, for printing.
"""
# Loading opt files
if load_raw:
with open(opt_path_1) as f1:
opt1 = json.load(f1)
with open(opt_path_2) as f2:
opt2 = json.load(f2)
else:
opt1 = Opt.load(opt_path_1)
opt2 = Opt.load(opt_path_2)
outputs = list()
outputs.append('\nArgs only found in opt 1:')
opt1_only_keys = sorted([k for k in opt1.keys() if k not in opt2.keys()])
for key in opt1_only_keys:
outputs.append(f'{key}: {opt1[key]}')
outputs.append('\nArgs only found in opt 2:')
opt2_only_keys = sorted([k for k in opt2.keys() if k not in opt1.keys()])
for key in opt2_only_keys:
outputs.append(f'{key}: {opt2[key]}')
outputs.append('\nArgs that are different in both opts:')
keys_with_conflicting_values = sorted(
[k for k, v in opt1.items() if k in opt2.keys() and v != opt2[k]]
)
for key in keys_with_conflicting_values:
if isinstance(opt1[key], dict) and isinstance(opt2[key], dict):
outputs.append(f'{key} (printing only non-matching values in each dict):')
all_inner_keys = sorted(
list(set(opt1[key].keys()).union(set(opt2[key].keys())))
)
for inner_key in all_inner_keys:
if (
inner_key not in opt1[key]
or inner_key not in opt2[key]
or opt1[key][inner_key] != opt2[key][inner_key]
):
outputs.append(f'\t{inner_key}:')
outputs.append(
f'\t\tIn opt 1: {opt1[key].get(inner_key, "<MISSING>")}'
)
outputs.append(
f'\t\tIn opt 2: {opt2[key].get(inner_key, "<MISSING>")}'
)
else:
outputs.append(f'{key}:')
outputs.append(f'\tIn opt 1: {opt1[key]}')
outputs.append(f'\tIn opt 2: {opt2[key]}')
return '\n'.join(outputs)
def compare_init_model_opts(opt: Opt, curr_opt: Opt):
"""
Print loud warning when `init_model` opts differ from previous configuration.
"""
if opt.get('init_model') is None:
return
opt['init_model'] = modelzoo_path(opt['datapath'], opt['init_model'])
optfile = opt['init_model'] + '.opt'
if not os.path.isfile(optfile):
return
init_model_opt = Opt.load(optfile)
extra_opts = {}
different_opts = {}
exempt_opts = [
'model_file',
'dict_file',
'override',
'starttime',
'init_model',
'batchindex',
]
# search through init model opts
for k, v in init_model_opt.items():
if (k not in exempt_opts and k in init_model_opt
and init_model_opt[k] != curr_opt.get(k)):
if isinstance(v, list):
if init_model_opt[k] != list(curr_opt[k]):
different_opts[k] = ','.join([str(x) for x in v])
else:
different_opts[k] = v
# search through opts to load
for k, v in curr_opt.items():
if k not in exempt_opts and k not in init_model_opt:
if isinstance(v, list):
extra_opts[k] = ','.join([str(x) for x in v])
else:
extra_opts[k] = v
# print warnings
extra_strs = ['{}: {}'.format(k, v) for k, v in extra_opts.items()]
if extra_strs:
print('\n' + '*' * 75)
print('[ WARNING ] : your model is being loaded with opts that do not '
'exist in the model you are initializing the weights with: '
'{}'.format(','.join(extra_strs)))
different_strs = [
'--{} {}'.format(k, v).replace('_', '-')
for k, v in different_opts.items()
]
if different_strs:
print('\n' + '*' * 75)
print('[ WARNING ] : your model is being loaded with opts that differ '
'from the model you are initializing the weights with. Add the '
'following args to your run command to change this: \n'
'\n{}'.format(' '.join(different_strs)))
print('*' * 75)
def _get_build_options(cls, opt: Opt):
"""
Return build options for DprEncoders.
:return (query_model, query_path, document_model, document_path):
query_model: dpr query model type
query_path: path to pre-trained DPR model
document_model: dpr document model type
document_path: path to pre-trained document model
"""
query_model = 'bert'
document_model = 'bert'
query_path = opt['model_file']
document_path = opt['model_file']
try:
# determine if loading a RAG model
loaded_opt = Opt.load(f"{query_path}.opt")
if loaded_opt['model'] == 'rag' and loaded_opt['query_model'] in [
'bert',
'bert_from_parlai_rag',
]:
query_model = 'bert_from_parlai_rag'
# document model is always frozen
document_path = loaded_opt.get('dpr_model_file', document_path)
except FileNotFoundError:
pass
return query_model, query_path, document_model, document_path
def test_save_load(self):
o = Opt({'a': 3, 'b': 'foo'})
with testing_utils.tempdir() as tmpdir:
fn = os.path.join(tmpdir, "opt")
o.save(fn)
o2 = Opt.load(fn)
assert o == o2
def create_agent_from_opt_file(opt: Opt):
"""
Load agent options and module from file if opt file exists.
Checks to see if file exists opt['model_file'] + ".opt"; if so, load up the
options from the file and use that to create an agent, loading the model
type from that file and overriding any options specified in that file when
instantiating the agent.
If that file does not exist, return None.
"""
model_file = opt['model_file']
optfile = model_file + '.opt'
if os.path.isfile(optfile):
new_opt = Opt.load(optfile)
# TODO we need a better way to say these options are never copied...
if 'datapath' in new_opt:
# never use the datapath from an opt dump
del new_opt['datapath']
if 'batchindex' in new_opt:
# This saved variable can cause trouble if we switch to BS=1 at test time
del new_opt['batchindex']
# only override opts specified in 'override' dict
if opt.get('override'):
for k, v in opt['override'].items():
if k in new_opt and str(v) != str(new_opt.get(k)):
logging.warn(
f"overriding opt['{k}'] to {v} (previously: {new_opt.get(k)})"
)
new_opt[k] = v
model_class = load_agent_module(new_opt['model'])
if hasattr(model_class, 'upgrade_opt'):
new_opt = model_class.upgrade_opt(new_opt)
# add model arguments to new_opt if they aren't in new_opt already
for k, v in opt.items():
if k not in new_opt:
new_opt[k] = v
new_opt['model_file'] = model_file
if not new_opt.get('dict_file'):
new_opt['dict_file'] = model_file + '.dict'
elif new_opt.get('dict_file') and not os.path.isfile(
new_opt['dict_file']):
old_dict_file = new_opt['dict_file']
new_opt['dict_file'] = model_file + '.dict'
if not os.path.isfile(new_opt['dict_file']):
warn_once(
'WARNING: Neither the specified dict file ({}) nor the '
'`model_file`.dict file ({}) exists, check to make sure either '
'is correct. This may manifest as a shape mismatch later '
'on.'.format(old_dict_file, new_opt['dict_file']))
# if we want to load weights from --init-model, compare opts with
# loaded ones
compare_init_model_opts(opt, new_opt)
return model_class(new_opt)
else:
return None
def _build_model(self, opt: Opt) -> Tuple[PolyEncoderModule, DictionaryAgent]:
"""
Build poly-encoder module.
:param opt:
options from base RAG Model
:return dropout poly-encoder:
return dropout poly agent.
"""
model_file = modelzoo_path(opt['datapath'], opt['poly_faiss_model_file'])
model_opt = Opt.load(f'{model_file}.opt')
create_model_opt = {
**{k: model_opt[k] for k in TRANSFORMER_RANKER_BASE_OPT},
**{k: model_opt[k] for k in POLYENCODER_OPT_KEYS},
'model': 'transformer/dropout_poly',
'init_model': model_file,
'dict_file': f'{model_file}.dict',
# necessary opt args
'multitask_weights': [1],
# dropout_poly args
'poly_dropout_reduction_type': model_opt['poly_dropout_reduction_type'],
'poly_dropout_use_codes': model_opt.get('poly_dropout_use_codes', True),
}
logging.disable()
agent = create_agent(Opt(create_model_opt))
logging.enable()
assert isinstance(agent, DropoutPolyAgent)
return agent.model, agent.dict
def __init__(self, opt: Opt):
super().__init__()
self.model, self.dict = self._build_model(opt)
model_file = modelzoo_path(opt['datapath'], opt['poly_faiss_model_file'])
model_opt = Opt.load(f"{model_file}.opt")
self.reduction_type = model_opt['poly_dropout_reduction_type']
self.use_codes = model_opt.get('poly_dropout_use_codes', True)
def _init_attributes(self, opt: Opt):
"""
Given opt dictionary, initialize relevant attributes for the predictor.
:param opt:
options dict
"""
optfile = f"{self.predictor_model_file}.opt"
opt_from_file = Opt.load(optfile)
overrides = opt.get('override')
opt_from_file.update(overrides)
assert 'num_utterances' in opt_from_file, opt_from_file
self.num_utterances = opt_from_file['num_utterances']
self.annotate_speaker = opt_from_file['annotate_speaker']
self.speaker_annotation_position = opt_from_file[
'speaker_annotation_position']
self.speaker_label_type = opt_from_file['speaker_label_type']
self.speaker_separator = opt_from_file['speaker_separator']
self.include_context = opt_from_file['include_light_context']
if opt_from_file.get('exclude_from_context'):
self.exclude_from_context = opt_from_file[
'exclude_from_context'].split(',')
else:
self.exclude_from_context = []
def build_regret_model(self) -> RagModel:
"""
Build and return regret RagModel.
Assume dictionary is the same.
"""
model_file = self.opt['regret_model_file']
if model_file:
assert os.path.exists(
model_file), 'specify correct path for --regret-model-file'
regret_opt = Opt.load(f'{model_file}.opt')
regret_opt['n_docs'] = self.opt[
'n_docs'] # Urgent that this is the same
# add keys that were not in this model when originally trained
regret_opt.update(
{k: v
for k, v in self.opt.items() if k not in regret_opt})
retriever_shared = None
if all([
regret_opt[k] == self.opt[k] for k in [
'rag_retriever_type',
'path_to_index',
'path_to_dpr_passages',
]
]):
logging.warning(
'Sharing retrievers between model and regret model!')
retriever_shared = self.model.encoder.retriever.share()
model = RagModel(regret_opt,
self.dict,
retriever_shared=retriever_shared)
with PathManager.open(self.opt['regret_model_file'], 'rb') as f:
states = torch.load(
f,
map_location=lambda cpu, _: cpu,
pickle_module=parlai.utils.pickle,
)
assert 'model' in states
model.load_state_dict(states['model'])
if self.model_parallel:
ph = PipelineHelper()
ph.check_compatibility(self.opt)
self.regret_model = ph.make_parallel(self.regret_model)
else:
self.regret_model.cuda()
if self.fp16:
self.regret_model = self.regret_model.half()
sync_parameters(self.regret_model)
train_params = trainable_parameters(self.regret_model)
total_params = total_parameters(self.regret_model)
logging.info(
f"Total regret parameters: {total_params:,d} ({train_params:,d} trainable)"
)
else:
model = self.model
return model
def test_save_withignore(self):
o = Opt({'a': 3, 'b': 'foo', 'override': {'a': 3}})
with testing_utils.tempdir() as tmpdir:
fn = os.path.join(tmpdir, "opt")
o.save(fn)
o2 = Opt.load(fn)
assert o != o2
assert 'override' not in o2
def create_agent_from_opt_file_and_model_class(opt, model_class):
model_file = opt['model_file']
optfile = model_file + '.opt'
if not PathManager.exists(optfile):
return None
opt_from_file = Opt.load(optfile)
# delete args that we do not want to copy over when loading the model
for arg in NOCOPY_ARGS:
if arg in opt_from_file:
del opt_from_file[arg]
# only override opts specified in 'override' dict
if opt.get('override'):
for k, v in opt['override'].items():
if k in opt_from_file and str(v) != str(opt_from_file.get(k)):
logging.warn(
f'Overriding opt["{k}"] to {v} (previously: {opt_from_file.get(k)})'
)
opt_from_file[k] = v
if hasattr(model_class, 'upgrade_opt'):
opt_from_file = model_class.upgrade_opt(opt_from_file)
# add model arguments to opt_from_file if they aren't in opt_from_file already
for k, v in opt.items():
if k not in opt_from_file:
opt_from_file[k] = v
# update model file path to the one set by opt
opt_from_file['model_file'] = model_file
# update init model path to the one set by opt
# NOTE: this step is necessary when for example the 'init_model' is
# set by the Train Loop (as is the case when loading from checkpoint)
if opt.get('init_model') is not None:
opt_from_file['init_model'] = opt['init_model']
# update dict file path
if not opt_from_file.get('dict_file'):
old_dict_file = None
opt_from_file['dict_file'] = model_file + '.dict'
elif opt_from_file.get('dict_file') and not PathManager.exists(
opt_from_file['dict_file']):
old_dict_file = opt_from_file['dict_file']
opt_from_file['dict_file'] = model_file + '.dict'
if not PathManager.exists(opt_from_file['dict_file']):
warn_once(
'WARNING: Neither the specified dict file ({}) nor the '
'`model_file`.dict file ({}) exists, check to make sure either '
'is correct. This may manifest as a shape mismatch later '
'on.'.format(old_dict_file, opt_from_file['dict_file']))
# if we want to load weights from --init-model, compare opts with
# loaded ones
compare_init_model_opts(opt, opt_from_file)
return model_class(opt_from_file)
def _get_model_name_from_model_file(self, key, opt):
"""
Get the model name from either `--model` or `--model-file`.
"""
# try to get model name from model opt file
model_file = opt.get(key, None)
optfile = model_file + ".opt"
new_opt = Opt.load(optfile)
model = new_opt.get("model", None)
return model
def _load_opts(self, opt):
optfile = opt.get('init_opt')
new_opt = Opt.load(optfile)
for key, value in new_opt.items():
# existing command line parameters take priority.
if key not in opt:
raise RuntimeError(
'Trying to set opt from file that does not exist: ' +
str(key))
if key not in opt['override']:
opt[key] = value
opt['override'][key] = value
def _load_known_opts(self, optfile, parsed):
"""
Pull in CLI args for proper models/tasks/etc.
Called before args are parsed; ``_load_opts`` is used for actually overriding
opts after they are parsed.
"""
new_opt = Opt.load(optfile)
for key, value in new_opt.items():
# existing command line parameters take priority.
if key not in parsed or parsed[key] is None:
parsed[key] = value
def get_model_name(opt):
"""
Get the model name from either `--model` or `--model-file`.
"""
model = opt.get('model', None)
if model is None:
# try to get model name from model opt file
model_file = opt.get('model_file', None)
if model_file is not None:
model_file = modelzoo_path(opt.get('datapath'), model_file)
optfile = model_file + '.opt'
if os.path.isfile(optfile):
new_opt = Opt.load(optfile)
model = new_opt.get('model', None)
return model
def _load_opts(self, opt):
optfile = opt.get('init_opt')
new_opt = Opt.load(optfile)
for key, value in new_opt.items():
# existing command line parameters take priority.
if key not in opt:
if opt.get('allow_missing_init_opts', False):
logging.warn(
f'The "{key}" key in {optfile} will not be loaded, because it '
f'does not exist in the target opt.')
else:
raise RuntimeError(
'Trying to set opt from file that does not exist: ' +
str(key))
if key not in opt['override']:
opt[key] = value
opt['override'][key] = value
def _get_subagent_opt(
self,
filename: str,
specific_override_args: Dict[str, Any],
general_override_args: Dict[str, Any],
) -> Opt:
"""
Given an agent opt, construct the new opt for the agent.
:param filename:
opt path
:param specific_override_args:
args for the specific agent
:param general_override_args:
args specified for all agents
"""
if not filename.endswith('.opt'):
filename += '.opt'
opt = Opt.load(modelzoo_path(self.opt['datapath'], filename))
opt['override'] = {}
blocklist_general = ['model', 'model_file', 'init_model']
general_override_args['skip_generation'] = False
# Remove the prefix for the model for the specific override args.
specific_override_args = {
'_'.join(k.split('_')[1:]): v for k, v in specific_override_args.items()
}
override_args = {**general_override_args, **specific_override_args}
for k, v in override_args.items():
if k not in blocklist_general and k in opt:
logging.warning(f'Overriding {k} to {v} (old val: {opt[k]})')
opt['override'][k] = v
elif k in specific_override_args:
logging.warning(f'Key {k} not originally in opt, setting to {v}')
opt['override'][k] = v
return opt
def get_bot_agents(
args: DictConfig,
active_models: Optional[List[str]] = None,
model_opts: Optional[Dict[str, str]] = None,
no_cuda=False,
) -> Dict[str, dict]:
"""
Return shared bot agents.
Pass in model opts in one of two ways: (1) With the `model_opts` arg, where
`model_opts` is a dictionary whose keys are model names and whose values are
strings that specify model params (i.e. `--model image_seq2seq`). (2) With the
`active_models` arg, a list of model names: those models' opts will be read
from args.blueprint.base_model_folder.
"""
# NOTE: in the future we may want to deprecate the `active_models` arg, to move
# away from the paradigm of having all models in one folder
model_overrides = {
'model_parallel': args.blueprint.task_model_parallel
}
if no_cuda:
# If we load many models at once, we have to keep it on CPU
model_overrides['no_cuda'] = no_cuda
else:
logging.warn(
'WARNING: MTurk task has no_cuda FALSE. Models will run on GPU. Will not work if loading many models at once.'
)
if active_models is not None:
model_overrides.update({
'datatype': 'valid', # So we don't have to load the optimizer
'encode_candidate_vecs':
True, # For pulling from fixed list cands
'interactive_mode': True,
'skip_generation': False,
})
# Add overrides that were historically used when reading models from a
# static folder
# Get the model nicknames from common folder and use them to load opts
# from file
base_model_folder = os.path.expanduser(
args.blueprint.base_model_folder)
models_available = []
for obj in os.listdir(base_model_folder):
if os.path.isdir(os.path.join(base_model_folder, obj)):
models_available.append(obj)
logging.info(
f'Found {len(models_available)} models available for Mturk task in {base_model_folder}: {models_available}'
)
all_model_opts = {}
logging.info(f'Active models to use are: {active_models}')
for model_nickname in active_models:
model_overrides_copy = copy.deepcopy(model_overrides)
model_path = os.path.join(base_model_folder, model_nickname,
'model')
if os.path.isfile(model_path):
model_opt = {
'model_file': model_path,
'override': model_overrides_copy,
}
else:
# Sometimes the model file is downloaded, like
# `-m hugging_face/dialogpt`
model_opt_path = model_path + '.opt'
logging.info(
f'Model file for model {model_nickname} does not exist! Instead, '
f'loading opt from {model_opt_path}.')
model_opt = Opt.load(model_opt_path)
if 'override' not in model_opt:
model_opt['override'] = {}
model_opt['override'].update(model_overrides_copy)
all_model_opts[model_nickname] = model_opt
final_model_opts = {m: all_model_opts[m] for m in active_models}
elif model_opts is not None:
parser = ParlaiParser(True, True)
parser.set_params(**model_overrides)
final_model_opts = {}
for name, opt in model_opts.items():
final_model_opts[name] = parser.parse_args(opt.split())
else:
raise ValueError(
'Either active_models or model_opts must be supplied!')
logging.info(
f'Got {len(list(final_model_opts.keys()))} active models with keys: {final_model_opts.keys()}.'
)
shared_bot_agents = {}
for model_name, model_opt in final_model_opts.items():
logging.info('\n\n--------------------------------')
logging.info(f'model_name: {model_name}, opt_dict: {model_opt}')
copied_opt_dict = copy.deepcopy(model_opt)
model_agent = create_agent(model_opt, requireModelExists=True)
if active_models is not None:
# have to check that the options are set properly
for k, v in copied_opt_dict.items():
if k != 'override':
assert model_agent.opt[k] == v
shared_bot_agents[model_name] = model_agent.share()
return shared_bot_agents
def get_bot_agents(opt: dict, active_models: list, no_cuda=False):
model_overrides = {
'datatype': 'valid', # So we don't have to load the optimizer
'encode_candidate_vecs': True, # For pulling from fixed list cands
'interactive_mode': True,
'model_parallel': opt['task_model_parallel'],
}
if no_cuda:
# If we load many models at once, we have to keep it on CPU
model_overrides['no_cuda'] = no_cuda
else:
logging.warn(
'WARNING: MTurk task has no_cuda FALSE. Models will run on GPU. Will not work if loading many models at once.'
)
# Get the model nicknames from common folder and use them to load opts
# from file, and add options specified in MODEL_CONFIGS
base_model_folder = opt.get('base_model_folder', None)
models_available = []
for obj in os.listdir(base_model_folder):
if os.path.isdir(os.path.join(base_model_folder, obj)):
models_available.append(obj)
print(
f'Found {len(models_available)} models available for Mturk task in {base_model_folder}: {models_available}'
)
all_model_opts = {}
print(f'Active models to use are: {active_models}')
for model_nickname in active_models:
model_overrides_copy = copy.deepcopy(model_overrides)
model_path = os.path.join(base_model_folder, model_nickname,
'model')
if os.path.isfile(model_path):
model_opt = {
'model_file': model_path,
'override': model_overrides_copy
}
else:
model_opt_path = model_path + '.opt'
print(
f'Model file for model {model_nickname} does not exist! Instead, '
f'loading opt from {model_opt_path}.')
model_opt = Opt.load(model_opt_path)
if 'override' not in model_opt:
model_opt['override'] = {}
model_opt['override'].update(model_overrides_copy)
all_model_opts[model_nickname] = model_opt
active_model_opt_dicts = {m: all_model_opts[m] for m in active_models}
print(
f'Got {len(list(active_model_opt_dicts.keys()))} active models with keys: {active_model_opt_dicts.keys()}.'
)
shared_bot_agents = {}
for model_name, model_opt in active_model_opt_dicts.items():
print('\n\n--------------------------------')
print(f'model_name: {model_name}, opt_dict: {model_opt}')
copied_opt_dict = copy.deepcopy(model_opt)
model_agent = create_agent(model_opt, requireModelExists=True)
# have to check that the options are set properly
for k, v in copied_opt_dict.items():
if k != 'override':
assert model_agent.opt[k] == v
shared_bot_agents[model_name] = model_agent.share()
return shared_bot_agents
def create_agent_from_opt_file(opt: Opt):
"""
Load agent options and module from file if opt file exists.
Checks to see if file exists opt['model_file'] + ".opt"; if so, load up the
options from the file and use that to create an agent, loading the model
type from that file and overriding any options specified in that file when
instantiating the agent.
If that file does not exist, return None.
"""
model_file = opt['model_file']
optfile = model_file + '.opt'
if not os.path.isfile(optfile):
return None
opt_from_file = Opt.load(optfile)
# delete args that we do not want to copy over when loading the model
for arg in NOCOPY_ARGS:
if arg in opt_from_file:
del opt_from_file[arg]
# only override opts specified in 'override' dict
if opt.get('override'):
for k, v in opt['override'].items():
if k in opt_from_file and str(v) != str(opt_from_file.get(k)):
logging.warn(
f'Overriding opt["{k}"] to {v} (previously: {opt_from_file.get(k)})'
)
opt_from_file[k] = v
model_class = load_agent_module(opt_from_file['model'])
if hasattr(model_class, 'upgrade_opt'):
opt_from_file = model_class.upgrade_opt(opt_from_file)
# add model arguments to opt_from_file if they aren't in opt_from_file already
for k, v in opt.items():
if k not in opt_from_file:
opt_from_file[k] = v
opt_from_file['model_file'] = model_file # update model file path
# update dict file path
if not opt_from_file.get('dict_file'):
opt_from_file['dict_file'] = model_file + '.dict'
elif opt_from_file.get('dict_file') and not os.path.isfile(
opt_from_file['dict_file']):
old_dict_file = opt_from_file['dict_file']
opt_from_file['dict_file'] = model_file + '.dict'
if not os.path.isfile(opt_from_file['dict_file']):
warn_once(
'WARNING: Neither the specified dict file ({}) nor the '
'`model_file`.dict file ({}) exists, check to make sure either '
'is correct. This may manifest as a shape mismatch later '
'on.'.format(old_dict_file, opt_from_file['dict_file']))
# if we want to load weights from --init-model, compare opts with
# loaded ones
compare_init_model_opts(opt, opt_from_file)
return model_class(opt_from_file)
def create_agent_from_opt_file(opt: Opt):
"""
Load agent options and module from file if opt file exists.
Checks to see if file exists opt['model_file'] + ".opt"; if so, load up the
options from the file and use that to create an agent, loading the model
type from that file and overriding any options specified in that file when
instantiating the agent.
If that file does not exist, return None.
"""
model_file = opt['model_file']
optfile = model_file + '.opt'
if os.path.isfile(optfile):
new_opt = Opt.load(optfile)
# TODO we need a better way to say these options are never copied...
if 'datapath' in new_opt:
# never use the datapath from an opt dump
del new_opt['datapath']
if 'batchindex' in new_opt:
# This saved variable can cause trouble if we switch to BS=1 at test time
del new_opt['batchindex']
# only override opts specified in 'override' dict
if opt.get('override'):
for k, v in opt['override'].items():
if str(v) != str(new_opt.get(k, None)):
print(
"[ warning: overriding opt['{}'] to {} ("
"previously: {} )]".format(k, v, new_opt.get(k, None))
)
new_opt[k] = v
model_class = load_agent_module(new_opt['model'])
# check for model version
if hasattr(model_class, 'model_version'):
curr_version = new_opt.get('model_version', 0)
if curr_version != model_class.model_version():
model = new_opt['model']
m = (
'It looks like you are trying to load an older version of'
' the selected model. Change your model argument to use '
'the old version from parlai/agents/legacy_agents: for '
'example: `-m legacy:{m}:{v}` or '
'`--model parlai.agents.legacy_agents.{m}.{m}_v{v}:{c}`'
)
if '.' not in model:
# give specific error message if it's easy
raise RuntimeError(
m.format(m=model, v=curr_version, c=model_class.__name__)
)
else:
# otherwise generic one
raise RuntimeError(
m.format(m='modelname', v=curr_version, c='ModelAgent')
)
if hasattr(model_class, 'upgrade_opt'):
new_opt = model_class.upgrade_opt(new_opt)
# add model arguments to new_opt if they aren't in new_opt already
for k, v in opt.items():
if k not in new_opt:
new_opt[k] = v
new_opt['model_file'] = model_file
if not new_opt.get('dict_file'):
new_opt['dict_file'] = model_file + '.dict'
elif new_opt.get('dict_file') and not os.path.isfile(new_opt['dict_file']):
old_dict_file = new_opt['dict_file']
new_opt['dict_file'] = model_file + '.dict'
if not os.path.isfile(new_opt['dict_file']):
warn_once(
'WARNING: Neither the specified dict file ({}) nor the '
'`model_file`.dict file ({}) exists, check to make sure either '
'is correct. This may manifest as a shape mismatch later '
'on.'.format(old_dict_file, new_opt['dict_file'])
)
# if we want to load weights from --init-model, compare opts with
# loaded ones
compare_init_model_opts(opt, new_opt)
return model_class(new_opt)
else:
return None
def initialize():
opt = Opt.load('opt')
global root_agent
root_agent = create_agent(opt, requireModelExists=True)
