def main():
freeze_support()
parser = create_main_args_parser()
args = parser.parse_args()
params = ()
for param_name in ['dir1', 'dir2']:
if param_name in args:
params += (args.__getattribute__(param_name), )
init_program(params[0])
args.command(*params)
def improve(self):
freeze_support()
os.environ['OMP_NUM_THREADS'] = '1'
os.environ['CUDA_VISIBLE_DEVICES'] = ""
processes = []
for _ in range(0, BatchA3C.NUM_PROCESS):
p = mp.Process(target=BatchA3C._improve,
args=(self.gloal_value_estimator,
self.global_policy, self.env,
self.num_episodes))
p.start()
processes.append(p)
for p in processes:
p.join()
output_feat=args.output_size,
seq_len=args.obs_seq_len,
kernel_size=args.kernel_size,
pred_seq_len=args.pred_seq_len,
hot_enc_length=len(config.labels)).cuda()
model.load_state_dict(torch.load(model_path))
model.cuda()
ade_ = 999999
fde_ = 999999
aade_ = 999999
afde_ = 999999
print("Testing ....")
ad, fd, aad, afd, raw_data_dic_ = test()
ade_ = min(ade_, ad)
fde_ = min(fde_, fd)
aade_ = min(aade_, aad)
afde_ = min(afde_, afd)
ade_ls.append(ade_)
fde_ls.append(fde_)
print("mADE:", ade_, " mFDE:", fde_)
print("aADE:", aade_, "aFDE:", afde_)
if __name__ == '__main__':
freeze_support()
main()
def get_flair_vectors(cls,
raw_sentences: Union[List[str], List[List[str]]],
flair_model_path: str,
flair_algorithm: str,
retrain_corpus_path: str = None,
epochs: int = 10):
freeze_support()
# retrain
if retrain_corpus_path:
if not os.path.isdir(retrain_corpus_path):
raw_sentences = cls.build_flair_corpus(raw_sentences,
retrain_corpus_path)
cls.retrain_flair(corpus_path=retrain_corpus_path,
model_path_dest=flair_model_path,
flair_algorithm=flair_algorithm,
epochs=epochs)
if os.path.exists(os.path.dirname(flair_model_path)):
flair_model_path = os.path.join(flair_model_path, 'best-lm.pt')
use_embedding, _ = cls.determine_algorithm_from_string(
flair_algorithm_string=flair_algorithm)
embedding = use_embedding(flair_model_path)
if any(isinstance(el, list) for el in raw_sentences):
use_tokenizer = False
raw_sentences = [
' '.join(raw_sentence) for raw_sentence in raw_sentences
if len(raw_sentence) > 0
]
else:
use_tokenizer = True
flair_sents = [
Sentence(raw_sentence, use_tokenizer=use_tokenizer)
for raw_sentence in tqdm(raw_sentences,
desc="Convert to flair",
total=len(raw_sentences))
if raw_sentence != '' and len(raw_sentence) > 0
]
flair_sents = [
flair_sent for flair_sent in flair_sents
if flair_sent and len(flair_sent) > 0
]
# keyed_vecs_o = defaultdict(list)
# for flair_sentence in tqdm(flair_sents, desc='Embed sentences', total=len(flair_sents)):
# embedding.embed(flair_sentence)
# for token in flair_sentence:
# keyed_vecs_o[token.text].append(token.embedding.cpu())
# keyed_vecs_o = {key: np.array(torch.mean(torch.stack(vecs), 0).cpu()) for key, vecs in keyed_vecs_o.items()}
keyed_vecs = {}
for flair_sentence in tqdm(flair_sents,
desc='Embed sentences',
total=len(flair_sents)):
try:
embedding.embed(flair_sentence)
except IndexError:
continue
for token in flair_sentence:
if token.text in keyed_vecs:
cur, inc = keyed_vecs[token.text]
new_token_embedding = token.embedding.cpu()
# print(len(np.array(new_token_embedding)))
if new_token_embedding.size() == cur.size():
keyed_vecs[token.text] = (cur +
(new_token_embedding - cur) /
(inc + 1), inc + 1)
else:
keyed_vecs[token.text] = (token.embedding.cpu(), 1)
flair_sentence.clear_embeddings()
keyed_vecs = {
key: np.array(vecs[0])
for key, vecs in keyed_vecs.items()
}
keyed_vecs = {
key: vecs
for key, vecs in keyed_vecs.items() if len(vecs) != 0
}
# for key, vec in keyed_vecs.items():
# if len(vec) != 3072:
# print(key, len(vec))
return Embeddings.to_gensim_binary(keyed_vecs)
topic,n = cmd_sub.recv()
if "notify.eye_process.should_start" in topic:
eye_id = n['eye_id']
if not eyes_are_alive[eye_id].value:
Process(target=eye, name='eye{}'.format(eye_id), args=(
timebase,
eyes_are_alive[eye_id],
ipc_pub_url,
ipc_sub_url,
ipc_push_url,
user_dir,
app_version,
eye_id, glint_queue, glint_vector_queue
)).start()
elif "notify.launcher_process.should_stop" in topic:
break
elif "notify.meta.should_doc" in topic:
cmd_push.notify({
'subject':'meta.doc',
'actor':'launcher',
'doc':launcher.__doc__})
for p in active_children(): p.join()
if __name__ == '__main__':
freeze_support()
if platform.system() == 'Darwin':
set_start_method('spawn')
launcher()
import model
import loss
from option import args
from checkpoint import Checkpoint
from trainer import Trainer
print("main scale >>"+str(args.scale[0]))
utility.set_seed(args.seed)
checkpoint = Checkpoint(args)
if checkpoint.ok:
loader = data.Data(args)
model = model.Model(args, checkpoint)
loss = loss.Loss(args, checkpoint) if not args.test_only else None
t = Trainer(args, loader, model, loss, checkpoint)
def main():
while not t.terminate():
t.train()
t.test()
checkpoint.done()
if __name__ == '__main__': # 중복 방지를 위한 사용
freeze_support() # 윈도우에서 파이썬이 자원을 효율적으로 사용하게 만들어준다.
main()
