def run_model(task=None, model_name=None, dataset_name=None, config_file=None,
save_model=True, train=True, other_args=None):
"""
Args:
task(str): task name
model_name(str): model name
dataset_name(str): dataset name
config_file(str): config filename used to modify the pipeline's
settings. the config file should be json.
save_model(bool): whether to save the model
train(bool): whether to train the model
other_args(dict): the rest parameter args, which will be pass to the Config
"""
# load config
config = ConfigParser(task, model_name, dataset_name,
config_file, other_args)
# logger
logger = get_logger(config)
logger.info('Begin pipeline, task={}, model_name={}, dataset_name={}'.
format(str(task), str(model_name), str(dataset_name)))
# 加载数据集
dataset = get_dataset(config)
# 转换数据，并划分数据集
train_data, valid_data, test_data = dataset.get_data()
data_feature = dataset.get_data_feature()
'''
#add by 18231216
import torch
towr = open("oup.txt",'w')
for i,ement in enumerate(train_data):
for key in ement.data:
print("key:{},body:{}".format(key,torch.Tensor(ement.data[key])),file = towr)
print("#####################################",file = towr)
towr.close()
#'''
'''
#add by 18231216
towr = open("oup.txt",'w')
for key in enumerate(dataset.data):
print("key:{},shape:{}".format(key,torch.Tensor(dataset.data.data[key]).shape),file = towr)
towr.close()
'''
# 加载执行器
model_cache_file = './trafficdl/cache/model_cache/{}_{}.m'.format(
model_name, dataset_name)
model = get_model(config, data_feature)
executor = get_executor(config, model)
# 训练
if train or not os.path.exists(model_cache_file):
executor.train(train_data, valid_data)
if save_model:
executor.save_model(model_cache_file)
else:
executor.load_model(model_cache_file)
# 评估，评估结果将会放在 cache/evaluate_cache 下
executor.evaluate(test_data)
def run_model(task=None,
model_name=None,
dataset_name=None,
config_file=None,
save_model=True,
train=True,
other_args=None):
"""
Args:
task (str): task name
model_name (str): model name
dataset_name (str): dataset name
config_file (str): config filename used to modify the pipeline's
settings. the config file should be json.
save_model (bool): whether to save the model
train (bool): whether to train the model
other_args (dict): the rest parameter args, which will be pass to
the Config
"""
# load config
config = ConfigParser(task, model_name, dataset_name, config_file,
other_args)
# logger
logger = get_logger(config)
logger.info(
'Begin pipeline, task={}, model_name={}, dataset_name={}'.format(
str(task), str(model_name), str(dataset_name)))
# 加载数据集
dataset = get_dataset(config)
# 转换数据，并划分数据集
train_data, valid_data, test_data = dataset.get_data()
data_feature = dataset.get_data_feature()
# 加载执行器
model_cache_file = './trafficdl/cache/model_cache/{}_{}.m'.format(
model_name, dataset_name)
model = get_model(config, data_feature)
executor = get_executor(config, model)
# 训练
if train or not os.path.exists(model_cache_file):
executor.train(train_data, valid_data)
if save_model:
executor.save_model(model_cache_file)
else:
executor.load_model(model_cache_file)
# 评估，评估结果将会放在 cache/evaluate_cache 下
executor.evaluate(test_data)
100,
'clip_grad_norm':
False,
'use_early_stop':
True,
'max_grad_norm':
5,
'patience':
15,
}
os.environ["CUDA_VISIBLE_DEVICES"] = config['gpu_id']
config['device'] = torch.device(
"cuda" if torch.cuda.is_available() and config['gpu'] else "cpu")
logger = get_logger(config)
# 加载数据集
dataset = get_dataset(config)
# 转换数据，并划分数据集
train_data, valid_data, test_data = dataset.get_data()
print(len(train_data), len(train_data.dataset), train_data.dataset[0][0].shape,
train_data.dataset[0][1].shape, train_data.batch_size)
print(len(valid_data), len(valid_data.dataset), valid_data.dataset[0][0].shape,
valid_data.dataset[0][1].shape, valid_data.batch_size)
print(len(test_data), len(test_data.dataset), test_data.dataset[0][0].shape,
test_data.dataset[0][1].shape, test_data.batch_size)
data_feature = dataset.get_data_feature()
print(data_feature['adj_mx'].shape)
print(data_feature['adj_mx'].sum())