def main():
##Parameters = dataBase name, table name, file name
data = dataReader('TrafficData', 'nycData', 'NYCTrafficVolumeCounts.csv')
###Cross Bronx Expressway
cbe = data.query("""
select * from nycData where [Roadway Name] = 'Cross Bronx Expressway'
""")
crossBronxExpressway = data.extract(cbe)
###data graph on Cross Bronx Expressway
data.scatter(crossBronxExpressway, "Cross Bronx Expressway")
cbeVals = data.createList(crossBronxExpressway)
avgCbe = data.maxAverageValue(cbeVals)
print("Average of top 5 hours for traffic counts\nCross Brox Expressway: ")
print(avgCbe)
###
e42 = data.query("""
select * from nycData where [Roadway Name] = 'FDR Drive'
and [from] = 'East 42 Street'
and [to] = 'East 43 Street'
""")
east42 = data.extract(e42)
data.scatter(east42, "East 42nd Street")
east42Vals = data.createList(east42)
avgEast42 = data.maxAverageValue(east42Vals)
print("")
print("Average of the top 5 traffic counts\nEast 42nd Street")
print(avgEast42)
def infer(args, best=True, criterion=nn.CrossEntropyLoss()):
data_dir = args.data_dir
run_id = args.run_id
params_path = os.path.join(os.getcwd(), args.output_dir, args.run_id,
"log", "params.txt")
params_file = open(params_path, "r")
params = {}
for line in params_file:
temp = line.split()
params[str(temp[0])] = temp[2]
load_args(args, params)
setattr(args, "data_dir", data_dir)
checkpoint_name = "best_checkpoint"
if not best:
checkpoint_name = "last_checkpoint"
checkpoint_path = os.path.join(os.getcwd(), args.output_dir, args.run_id,
"save", checkpoint_name)
print checkpoint_path
if not os.path.exists(checkpoint_path):
raise ValueError(
"No checkpoint found. Please train the model before testing")
all_datasets = dataReader(args)
train_data = all_datasets['train']
val_data = all_datasets['validation']
test_data = all_datasets['test']
model = load_model(args)
if HAVE_CUDA:
model = model.cuda()
loss, accuracy, iou = get_validation_loss(args,
train_data,
model,
criterion,
infer=True)
print loss, accuracy, iou
loss, accuracy, iou = get_validation_loss(args,
val_data,
model,
criterion,
infer=True)
print loss, accuracy, iou
loss, accuracy, iou = get_validation_loss(args,
test_data,
model,
criterion,
infer=True)
print loss, accuracy, iou
return loss
def test(args, best=True, criterion=nn.CrossEntropyLoss()):
checkpoint_name = "best_checkpoint"
if not best:
checkpoint_name = "last_checkpoint"
checkpoint_path = os.path.join(os.getcwd(), args.output_dir, args.run_id,
"save", checkpoint_name)
print checkpoint_path
if not os.path.exists(checkpoint_path):
raise ValueError(
"No checkpoint found. Please train the model before testing")
all_datasets = dataReader(args)
test_data = all_datasets['test']
model = load_model(args)
if HAVE_CUDA:
model = model.cuda()
test_loss, accuracy, iou = get_validation_loss(args, test_data, model,
criterion)
print test_loss, accuracy, iou
return test_loss
def analyze():
analyzeData = dataReader('TrafficData', 'nycData',
'NYCTrafficVolumeCounts.csv')
successGroup = analyzeData.query("""
select * from nycData where [8:00-9:00AM] < '3000'
""")
failGroup = analyzeData.query("""
select * from nycData where [8:00-9:00AM] > '3000'
""")
successGroupTotal = 0
failGroupTotal = 0
fig = plt.figure()
fig.subplots_adjust(top=0.8)
ax1 = fig.add_subplot(211)
ax1.set_ylabel("# of Cars")
ax1.set_xlabel("Day/Street")
ax1.set_title("Traffic Counts vs. Model Speed")
for i in successGroup['8:00-9:00AM']:
successGroupTotal += 1
print(successGroupTotal)
ax1.scatter(successGroupTotal, i)
for i in failGroup['8:00-9:00AM']:
failGroupTotal += 1
ax1.scatter((successGroupTotal + failGroupTotal), i)
plt.show()
print(successGroupTotal)
print(failGroupTotal)
total = successGroupTotal + failGroupTotal
successRate = (successGroupTotal / total) * 100
print(successRate)
def dtrain2(args):
# Dataset loading and preparing env
all_datasets = dataReader(args)
train_data = all_datasets['train']
validation_data = all_datasets['validation']
model = UNet(num_classes=3)
# model = UNetj(3, 3)
# model = UNetm(3,3)
for batch in train_data:
print batch[0][0]
print model(ag.Variable(batch[0].float()))[0][0][0:5]
# print model(ag.Variable(batch[0].float()))[1][0][0][0:5]
break
criterion = nn.MSELoss()
optimizer = optim.Adam(model.parameters(), lr=0.01)
for i in range(100):
model, optimizer, present_train_loss = dtrain_epoch2(
args, train_data, model, optimizer, criterion)
if i % 10 == 0:
print "Loss", present_train_loss
for batch in train_data:
print batch[0][0]
print model(ag.Variable(batch[0].float()))[0][0][0:5]
break
def train(args):
# Dataset loading and preparing env
all_datasets = dataReader(args)
train_data = all_datasets['train']
validation_data = all_datasets['validation']
logger = ""
best_train_loss = float("Inf")
best_validation_loss = float("Inf")
best_epoch = -1
# instantiate model and optimizer
model = load_model(args)
model.train()
if HAVE_CUDA:
model = model.cuda()
criterion = nn.CrossEntropyLoss()
optimizer = load_optimizer(args, model)
# log info
prepare_env(args)
output_args(args)
loss_vals = []
train_acc_vals = []
val_acc_vals = []
train_iou_vals = []
val_iou_vals = []
train_start_time = time.time()
epoch_start_time = train_start_time
for i in range(args.num_epochs):
# run model
print "==========================================================================================="
print "Epoch(", i, "/", args.num_epochs, ")"
model, optimizer, present_train_loss, train_accuracy, train_iou, train_loss_vals = train_epoch(
args, train_data, model, optimizer, criterion)
loss_vals.extend(train_loss_vals)
present_validation_loss, validation_accuracy, validation_iou = get_validation_loss(
args, validation_data, model, criterion)
train_acc_vals.append(train_accuracy)
val_acc_vals.append(validation_accuracy)
train_iou_vals.append(train_iou)
val_iou_vals.append(validation_iou)
save_data(args, model, optimizer)
if present_validation_loss < best_validation_loss:
save_data(args, model, optimizer, True)
best_epoch = i
best_validation_loss = present_validation_loss
##########################################Logging info#############################################
end_time = time.time()
epoch_duration = end_time - epoch_start_time
epoch_start_time = end_time
print "Duration: ", epoch_duration
print ""
print "Loss:"
print "Training ", present_train_loss
print "Validation", present_validation_loss
print ""
print "Accuracy:"
print "Training ", train_accuracy
print "Validation", validation_accuracy
print ""
print "IoU:"
print "Training ", train_iou
print "Validation", validation_iou
print ""
print "Best Epoch", best_epoch
###################################################################################################
# break
save_data(args, model.cpu(), optimizer)
loss_vals = np.squeeze(np.array(loss_vals))
train_acc_vals = np.squeeze(np.array(train_acc_vals))
val_acc_vals = np.squeeze(np.array(val_acc_vals))
loss_save_path = os.path.join(os.getcwd(), args.output_dir, args.run_id,
"save", "loss")
np.save(loss_save_path, loss_vals)
train_save_path = os.path.join(os.getcwd(), args.output_dir, args.run_id,
"save", "train_accuracy")
np.save(train_save_path, train_acc_vals)
val_save_path = os.path.join(os.getcwd(), args.output_dir, args.run_id,
"save", "val_accuracy")
np.save(val_save_path, val_acc_vals)
train_save_path = os.path.join(os.getcwd(), args.output_dir, args.run_id,
"save", "train_iou")
np.save(train_save_path, train_iou_vals)
val_save_path = os.path.join(os.getcwd(), args.output_dir, args.run_id,
"save", "val_iou")
np.save(val_save_path, val_iou_vals)
total_training_time = time.time() - train_start_time
print "Total training time: ", total_training_time
def check(args):
data = dataReader(args)
for i, d in enumerate(data['train']):
print d[0][0]
break