def main(args):
config_filename = './data/model/dcrnn_{}.yaml'.format(args.city)
with open(config_filename) as f:
supervisor_config = yaml.load(f)
graph_pkl_filename = supervisor_config['data'].get(
'graph_pkl_filename')
adj_mx = load_graph_data1(graph_pkl_filename)
tf_config = tf.ConfigProto()
if args.use_cpu_only:
tf_config = tf.ConfigProto(device_count={'GPU': -1})
else:
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ['CUDA_VISIBLE_DEVICES'] = str(args.cuda)
print('Using GPU {}'.format(os.environ['CUDA_VISIBLE_DEVICES']))
tf_config.gpu_options.allow_growth = True
with tf.Session(config=tf_config) as sess:
supervisor = DCRNNSupervisor(adj_mx=adj_mx, **supervisor_config)
supervisor.train(sess=sess)
with tf.Session(config=tf_config) as sess:
supervisor = DCRNNSupervisor(adj_mx=adj_mx, **supervisor_config)
supervisor.load(sess, supervisor_config['train']['model_filename'])
outputs = supervisor.evaluate(sess)
def main(args):
with open(args.config_filename) as f:
supervisor_config = yaml.load(f)
SC_mx = load_graph_data(
supervisor_config) # Load structural connectivity matrix.
tf_config = tf.ConfigProto()
if args.use_cpu_only:
tf_config = tf.ConfigProto(device_count={'GPU': 0})
tf_config.gpu_options.allow_growth = True
with tf.Session(config=tf_config) as sess:
supervisor = DCRNNSupervisor(adj_mx=SC_mx, **supervisor_config)
supervisor.train(sess=sess)
if args.save_predictions:
outputs, _ = supervisor.evaluate(sess=sess)
print('Save outputs in: ', supervisor._log_dir)
np.savez(supervisor._log_dir + '/outputs',
predictions=outputs['predictions'],
groundtruth=outputs['groundtruth'])
plot_predictions(
log_dir=supervisor._log_dir,
dataset_dir=supervisor_config['data']['dataset_dir'])
def main(args):
#Pick GPU to use. Activate tensorflow_gpu conda env
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_instance
### Open model parameters file
with open(args.config_filename) as f:
supervisor_config = yaml.load(f)
### Load adjacency matrix
graph_pkl_filename = supervisor_config['data'].get('graph_pkl_filename')
### load the graph look at /lib/utils.py for this function
sensor_ids, sensor_id_to_ind, adj_mx = load_graph_data(graph_pkl_filename)
tf_config = tf.ConfigProto()
if args.use_cpu_only:
tf_config = tf.ConfigProto(device_count={'GPU': 0})
tf_config.gpu_options.allow_growth = True
### Call the DCRNN supervisor class and start training
with tf.Session(config=tf_config) as sess:
supervisor = DCRNNSupervisor(adj_mx=adj_mx, **supervisor_config)
supervisor.train(sess=sess)
def main():
# Reads graph data.
with open(FLAGS.config_filename) as f:
supervisor_config = json.load(f)
logger = log_helper.get_logger(supervisor_config.get('base_dir'),
'info.log')
logger.info('Loading graph from: ' + FLAGS.graph_pkl_filename)
sensor_ids, sensor_id_to_ind, adj_mx = load_graph_data(
FLAGS.graph_pkl_filename)
adj_mx[adj_mx < 0.1] = 0
logger.info('Loading traffic data from: ' + FLAGS.traffic_df_filename)
traffic_df_filename = FLAGS.traffic_df_filename
traffic_reading_df = pd.read_csv(traffic_df_filename)
#modify by AG
sensors_ids = [
'time_stamp', 'aqi_W San Gabriel Vly', 'aqi_E San Fernando Vly',
'aqi_SW Coastal LA', 'aqi_San Gabriel Mts',
'aqi_SW San Bernardino', 'aqi_Southeast LA CO',
'aqi_South Coastal LA', 'aqi_Central LA CO', 'aqi_NW Coastal LA',
'aqi_Santa Clarita Vly', 'aqi_W San Fernando Vly',
'aqi_E San Gabriel V-2'
]
#test
test = pd.read_hdf('data/df_highway_2012_4mon_sample.h5')
test = test.ix[:, ]
traffic_reading_df = traffic_reading_df.ix[:, sensor_ids]
supervisor_config['use_cpu_only'] = FLAGS.use_cpu_only
if FLAGS.log_dir:
supervisor_config['log_dir'] = FLAGS.log_dir
if FLAGS.use_curriculum_learning is not None:
supervisor_config[
'use_curriculum_learning'] = FLAGS.use_curriculum_learning
if FLAGS.loss_func:
supervisor_config['loss_func'] = FLAGS.loss_func
if FLAGS.filter_type:
supervisor_config['filter_type'] = FLAGS.filter_type
# Overwrites space with specified parameters.
for name in [
'batch_size', 'cl_decay_steps', 'epochs', 'horizon',
'learning_rate', 'l1_decay', 'lr_decay', 'lr_decay_epoch',
'lr_decay_interval', 'learning_rate', 'min_learning_rate',
'patience', 'seq_len', 'test_every_n_epochs', 'verbose'
]:
if getattr(FLAGS, name) >= 0:
supervisor_config[name] = getattr(FLAGS, name)
tf_config = tf.ConfigProto()
if FLAGS.use_cpu_only:
tf_config = tf.ConfigProto(device_count={'GPU': 0})
tf_config.gpu_options.allow_growth = True
with tf.Session(config=tf_config) as sess:
supervisor = DCRNNSupervisor(traffic_reading_df=traffic_reading_df,
adj_mx=adj_mx,
config=supervisor_config)
supervisor.train(sess=sess)
def train_dcrnn(args, dataloaders, adj_mx):
if not args.test_only:
args = get_model_filename(args)
tf_config = setup_tf(args)
with tf.Session(config=tf_config) as sess:
supervisor = \
DCRNNSupervisor(sess, adj_mx, dataloaders, args)
supervisor.train(sess)
return args
def main(args):
with open(args.config_filename) as f:
supervisor_config = yaml.load(f)
SEED = 1234
random.seed(SEED)
np.random.seed(SEED)
tf.set_random_seed(SEED)
tf_config = tf.ConfigProto()
if args.use_cpu_only:
tf_config = tf.ConfigProto(device_count={'GPU': 0})
tf_config.gpu_options.allow_growth = True
bytes_in_use = BytesInUse()
maxbytes_in_use = tf.contrib.memory_stats.MaxBytesInUse()
with tf.Session(config=tf_config) as sess:
# Train
data_tag = supervisor_config.get('base_dir')
if restart:
data_tag = supervisor_config.get('base_dir')
yaml_files = glob.glob('%s/*/*.yaml' % data_tag,
recursive=True)
yaml_files.sort(key=os.path.getmtime)
config_filename = yaml_files[-1] #'config_%d.yaml' % config_id
print(config_filename)
with open(config_filename) as f:
config = yaml.load(f)
supervisor = DCRNNSupervisor(config)
supervisor.load(sess, config['train']['model_filename'])
else:
supervisor = DCRNNSupervisor(supervisor_config)
folder = data_tag
if not os.path.exists(folder):
os.makedirs(folder)
supervisor.train(sess=sess)
# Test
supervisor = DCRNNSupervisor(supervisor_config)
data_tag = supervisor_config.get('base_dir')
yaml_files = glob.glob('%s/*/*.yaml' % data_tag, recursive=True)
yaml_files.sort(key=os.path.getmtime)
config_filename = yaml_files[-1] #'config_%d.yaml' % config_id
with open(config_filename) as f:
config = yaml.load(f)
# Load model and evaluate
supervisor.load(sess, config['train']['model_filename'])
y_preds = supervisor.evaluate(sess)
folder = 'data/results/'
if not os.path.exists(folder):
os.makedirs(folder)
df_sp = pd.DataFrame(y_preds)
filename = os.path.join(folder, 'dcrnn_speed_bay.h5')
df_sp.to_hdf(filename, 'results')
def main(args):
with open(args.config_filename) as f:
supervisor_config = yaml.load(f)
tf_config = tf.ConfigProto()
# if args.use_cpu_only:
# tf_config = tf.ConfigProto(device_count={'GPU': 0})
tf_config.gpu_options.allow_growth = True
with tf.Session(config=tf_config) as sess:
supervisor = DCRNNSupervisor(**supervisor_config)
supervisor.train(sess=sess)
def main(args):
with open(args.config_filename) as f:
supervisor_config = yaml.load(f)
graph_pkl_filename = supervisor_config['data'].get('graph_pkl_filename')
sensor_ids, sensor_id_to_ind, adj_mx = load_graph_data(graph_pkl_filename)
tf_config = tf.ConfigProto()
if args.use_cpu_only:
tf_config = tf.ConfigProto(device_count={'GPU': 0})
tf_config.gpu_options.allow_growth = True
with tf.Session(config=tf_config) as sess:
supervisor = DCRNNSupervisor(adj_mx=adj_mx, **supervisor_config)
supervisor.train(sess=sess)
def main(args):
with open(args.config_filename) as f:
supervisor_config = yaml.load(f)
graph_pkl_filename = supervisor_config['data'].get(
'graph_pkl_filename')
sensor_ids, sensor_id_to_ind, adj_mx = load_graph_data(
graph_pkl_filename)
import os
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
with tf.Session(config=tf_config) as sess:
supervisor = DCRNNSupervisor(adj_mx=adj_mx, **supervisor_config)
supervisor.train(sess=sess)
def main():
# Reads graph data.
with open(FLAGS.config_filename) as f:
supervisor_config = yaml.load(f)
logger = log_helper.get_logger(supervisor_config.get('base_dir'),
'info.log')
logger.info('Loading graph from: ' + FLAGS.graph_pkl_filename)
sensor_ids, sensor_id_to_ind, adj_mx = load_graph_data(
FLAGS.graph_pkl_filename)
adj_mx[adj_mx < 0.1] = 0
logger.info('Loading traffic data from: ' + FLAGS.traffic_df_filename)
traffic_df_filename = FLAGS.traffic_df_filename
traffic_reading_df = pd.read_hdf(traffic_df_filename)
traffic_reading_df = traffic_reading_df.ix[:, sensor_ids]
supervisor_config['use_cpu_only'] = FLAGS.use_cpu_only
if FLAGS.log_dir:
supervisor_config['log_dir'] = FLAGS.log_dir
if FLAGS.use_curriculum_learning is not None:
supervisor_config[
'use_curriculum_learning'] = FLAGS.use_curriculum_learning
if FLAGS.loss_func:
supervisor_config['loss_func'] = FLAGS.loss_func
if FLAGS.filter_type:
supervisor_config['filter_type'] = FLAGS.filter_type
# Overwrites space with specified parameters.
for name in [
'batch_size', 'cl_decay_steps', 'epochs', 'horizon',
'learning_rate', 'l1_decay', 'lr_decay', 'lr_decay_epoch',
'lr_decay_interval', 'learning_rate', 'min_learning_rate',
'patience', 'seq_len', 'test_every_n_epochs', 'verbose'
]:
if getattr(FLAGS, name) >= 0:
supervisor_config[name] = getattr(FLAGS, name)
tf_config = tf.ConfigProto()
if FLAGS.use_cpu_only:
tf_config = tf.ConfigProto(device_count={'GPU': 0})
tf_config.gpu_options.allow_growth = True
with tf.Session(config=tf_config) as sess:
supervisor = DCRNNSupervisor(traffic_reading_df=traffic_reading_df,
adj_mx=adj_mx,
config=supervisor_config)
supervisor.train(sess=sess)
def main(args):
tf.reset_default_graph()
with open(args.config_filename) as f:
with tf.Graph().as_default() as g:
supervisor_config = yaml.load(f)
graph_pkl_filename = supervisor_config['data'].get(
'graph_pkl_filename')
if supervisor_config['data']['data_type'] == 'npz':
sensor_ids, sensor_id_to_ind, adj_mx = load_graph_data(
graph_pkl_filename)
elif supervisor_config['data']['data_type'] == 'csv':
adj_mx = load_graph_data_from_csv(
supervisor_config['data'].get('dataset_dir'))
tf_config = tf.ConfigProto()
if args.use_cpu_only:
tf_config = tf.ConfigProto(device_count={'GPU': 0})
tf_config.gpu_options.allow_growth = True
#tf_config.gpu_options.per_process_gpu_memory_fraction = 1
with tf.Session(config=tf_config) as sess:
supervisor = DCRNNSupervisor(args=args,
adj_mx=adj_mx,
**supervisor_config)
supervisor.train(sess=sess)
def train_dcrnn(adj_mx, config):
# with tf.device('/device:GPU:{}'.format(config['gpu'])):
dcrnn_supervisor = DCRNNSupervisor(adj_mx=adj_mx, **config)
dcrnn_supervisor.train(sess=session)
def main(args):
with open(args.config_filename) as f:
supervisor_config = yaml.load(f)
graph_pkl_filename = supervisor_config['data'].get(
'graph_pkl_filename')
sensor_ids, sensor_id_to_ind, adj_mx = load_graph_data(
graph_pkl_filename)
supervisor_config['model']['num_nodes'] = num_nodes = len(sensor_ids)
# Data preprocessing
traffic_df_filename = supervisor_config['data']['hdf_filename']
df_data = pd.read_hdf(traffic_df_filename)
#df_data = df_data.iloc[int(df_data.shape[0]/3):,:]
validation_ratio = supervisor_config.get('data').get(
'validation_ratio')
test_ratio = supervisor_config.get('data').get('test_ratio')
df_train, df_val, df_test = train_val_test_split(
df_data, val_ratio=validation_ratio, test_ratio=test_ratio)
batch_size = supervisor_config.get('data').get('batch_size')
val_batch_size = supervisor_config.get('data').get('val_batch_size')
test_batch_size = supervisor_config.get('data').get('test_batch_size')
horizon = supervisor_config.get('model').get('horizon')
seq_len = supervisor_config.get('model').get('seq_len')
scaler = StandardScaler(mean=df_train.values.mean(),
std=df_train.values.std())
data_train = generate_seq2seq_data(df_train, batch_size, seq_len,
horizon, num_nodes, 'train', scaler)
data_val = generate_seq2seq_data(df_val, val_batch_size, seq_len,
horizon, num_nodes, 'val', scaler)
data_train.update(data_val)
#data_train['scaler'] = scaler
data_test = generate_seq2seq_data(df_test, test_batch_size, seq_len,
horizon, num_nodes, 'test', scaler)
#data_test['scaler'] = scaler
tf_config = tf.ConfigProto()
if args.use_cpu_only:
tf_config = tf.ConfigProto(device_count={'GPU': 0})
tf_config.gpu_options.allow_growth = True
with tf.Session(config=tf_config) as sess:
supervisor = DCRNNSupervisor(adj_mx, data_train, supervisor_config)
data_tag = supervisor_config.get('data').get('dataset_dir')
folder = data_tag + '/model/'
if not os.path.exists(folder):
os.makedirs(folder)
# Train
supervisor.train(sess=sess)
# Test
yaml_files = glob.glob('%s/model/*/*.yaml' % data_tag,
recursive=True)
yaml_files.sort(key=os.path.getmtime)
config_filename = yaml_files[-1] #'config_%d.yaml' % config_id
with open(config_filename) as f:
config = yaml.load(f)
# Load model and evaluate
supervisor.load(sess, config['train']['model_filename'])
y_preds = supervisor.evaluate(sess, data_test)
n_test_samples = data_test['y_test'].shape[0]
folder = data_tag + '/results/'
if not os.path.exists(folder):
os.makedirs(folder)
for horizon_i in range(data_test['y_test'].shape[1]):
y_pred = scaler.inverse_transform(y_preds[:, horizon_i, :, 0])
eval_dfs = df_test[seq_len + horizon_i:seq_len + horizon_i +
n_test_samples]
df = pd.DataFrame(y_pred,
index=eval_dfs.index,
columns=eval_dfs.columns)
#df = pd.DataFrame(y_pred, columns=df_test.columns)
filename = os.path.join(
'%s/results/' % data_tag,
'dcrnn_speed_prediction_%s.h5' % str(horizon_i + 1))
df.to_hdf(filename, 'results')
print(
'Predictions saved as %s/results/dcrnn_prediction_[1-12].h5...'
% data_tag)