# initialize data and label paths
print("[INFO] loading images........")
train_frame_path = os.path.sep.join([args["dataset"], "train_frames/train"])
train_mask_path = os.path.sep.join([args["dataset"], "train_masks/train"])
val_frame_path = os.path.sep.join([args["dataset"], "val_frames/val"])
val_mask_path = os.path.sep.join([args["dataset"], "val_masks/val"])
input_size = [640, 640]
# instantiate datagenerator class
train_set = Dataloader(image_paths=train_frame_path,
mask_paths=train_mask_path,
image_size=input_size,
numclasses=num_of_Classes,
channels=[3, 3],
palette=palette,
seed=47)
val_set = Dataloader(image_paths=val_frame_path,
mask_paths=val_mask_path,
image_size=input_size,
numclasses=num_of_Classes,
channels=[3, 3],
palette=palette,
seed=47)
# build model
model = UNET.build((640, 640, 3),
num_of_Classes,
test_frame_path = os.path.sep.join([dataset_path, "test_frames/test"])
test_mask_path = os.path.sep.join([dataset_path, "test_masks/test"])
# initialise variables
No_of_train_images = len(os.listdir(dataset_path + '/train_frames/train'))
No_of_val_images = len(os.listdir(dataset_path + '/val_frames/val'))
print("Number of Training Images = {}".format(No_of_train_images))
print("Number of Validation Images = {}".format(No_of_val_images))
input_size = [256, 256]
# instantiate datagenerator class
train_set = Dataloader(image_paths=train_frame_path,
mask_paths=train_mask_path,
image_size=input_size,
numclasses=num_of_Classes,
channels=[3, 3],
palette=palette,
seed=47)
val_set = Dataloader(image_paths=val_frame_path,
mask_paths=val_mask_path,
image_size=input_size,
numclasses=num_of_Classes,
channels=[3, 3],
palette=palette,
seed=47)
test_set = Dataloader(image_paths=test_frame_path,
mask_paths=test_mask_path,
image_size=input_size,
"""
Created on Thu Oct 24 19:25:42 2019
@author: User
"""
import xgboost as xgb
from sklearn.model_selection import RandomizedSearchCV,GridSearchCV,ShuffleSplit
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
from dataLoader import Dataloader
import matplotlib.pyplot as plt
import seaborn as sns
#Load Data
dl = Dataloader(normalization=True, select_features=["speed_max", "speed_mean", "speed_median", "speed_std"])
X_train, y_train = dl.getTrain()
X_test, y_test = dl.getTest()
X_validate, y_validate = dl.getValidate()
print(X_train.shape)
print(y_train.shape)
classes = {
"walk":0,
"bike":1,
"bus":2,
"taxi/car": 3,
test_maskpath = os.path.sep.join([test_datapath, "test_masks/test"])
test_vector_path = os.path.sep.join([test_datapath, "test_vectors/test"])
# input size
input_size = [352, 352]
# import colour palette
df = pd.read_csv('classes.csv', ",", header=None)
palette = np.array(df.values, dtype=np.float)
num_of_Classes = palette.shape[0]
# instantiate datagenerator class
test_set = Dataloader(image_paths=test_imagepath,
mask_paths=test_maskpath,
vector_paths=test_vector_path,
image_size=input_size,
numclasses=num_of_Classes,
channels=[3, 3],
palette=palette,
seed=47)
# evaluation parameters
BS = 1
# initialize data generators with threading lock
testgen = threadsafe_iter(
test_set.data_gen(should_augment=False, batch_size=BS))
No_of_test_images = len(os.listdir(test_imagepath))
print("Number of Test Images = {}".format(No_of_test_images))
eval = evaluation(input_size,
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import classification_report, confusion_matrix
import warnings
warnings.filterwarnings('ignore')
from dataLoader import Dataloader
from sklearn.tree import export_graphviz
from sklearn.externals.six import StringIO
from IPython.display import Image
import pydotplus
import os
os.environ["PATH"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin/'
#Load Data
dl = Dataloader()
X = dl.getX()
X_train, y_train = dl.getTrain()
X_test, y_test = dl.getTest()
print(X_train.shape)
print(y_train.shape)
n_features = 33 #to edit if there's any features that are dropped
classes = dl.getClasses()
inv_map = {v: k for k, v in classes.items()}
print(classes)
plt.figure()
plt.imshow(img)
plt.axis('off')
plt.style.use('ggplot') # if want to use the default style, set 'classic'
plt.rcParams['ytick.right'] = True
plt.rcParams['ytick.labelright']= True
plt.rcParams['ytick.left'] = False
plt.rcParams['ytick.labelleft'] = False
plt.rcParams['font.family'] = 'Arial'
modelname = 'pre-1'
seed = 7
np.random.seed(seed)
# .............................................................................
dl = Dataloader(normalization=True, noise_removal=True)
x_train, y_train = dl.getTrain()
x_val, y_val = dl.getValidate()
#enc = OneHotEncoder(categories=[classes],handle_unknown='ignore',drop=[0])
y_train = to_categorical(y_train)
y_val = to_categorical(y_val)
x_train = np.expand_dims(x_train,axis=2)
x_val = np.expand_dims(x_val,axis=2)
dat = tf.convert_to_tensor(x_train)
lbl = tf.convert_to_tensor(y_train)
ds = tf.data.Dataset.from_tensor_slices((dat, lbl))
dataset = ds.shuffle(1000).batch(1).repeat()
set_seed(config['seed'])
if 'multiwoz' in data_dir:
print('-'*20 + 'dataset:multiwoz' + '-'*20)
from convlab2.nlu.jointBERT.multiwoz.postprocess import is_slot_da, calculateF1, recover_intent
elif 'camrest' in data_dir:
print('-' * 20 + 'dataset:camrest' + '-' * 20)
from convlab2.nlu.jointBERT.camrest.postprocess import is_slot_da, calculateF1, recover_intent
elif 'crosswoz' in data_dir:
print('-' * 20 + 'dataset:crosswoz' + '-' * 20)
from convlab2.nlu.jointBERT.crosswoz.postprocess import is_slot_da, calculateF1, recover_intent
intent_vocab = json.load(open(os.path.join(data_dir, 'intent_vocab.json')))
tag_vocab = json.load(open(os.path.join(data_dir, 'tag_vocab.json')))
dataloader = Dataloader(intent_vocab=intent_vocab, tag_vocab=tag_vocab,
pretrained_weights=config['model']['pretrained_weights'])
print('intent num:', len(intent_vocab))
print('tag num:', len(tag_vocab))
for data_key in ['train', 'val', 'test']:
dataloader.load_data(json.load(open(os.path.join(data_dir, '{}_data.json'.format(data_key)))), data_key,
cut_sen_len=config['cut_sen_len'], use_bert_tokenizer=config['use_bert_tokenizer'])
print('{} set size: {}'.format(data_key, len(dataloader.data[data_key])))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
writer = SummaryWriter(log_dir)
model = JointBERT(config['model'], DEVICE, dataloader.tag_dim, dataloader.intent_dim, dataloader.intent_weight)
from dataLoader import Dataloader from vincenty import vincenty points, labels, traj = Dataloader(load_portion=0.02).getDataFrames() print(points) print(labels) print(traj) data, lbl = Dataloader().getTrain() data, lbl = Dataloader().getTest() data, lbl = Dataloader().getValidate()
from dataLoader import Dataloader
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_selection import SelectFromModel
from tensorflow.python.keras.utils import to_categorical
x_train, y_train = Dataloader().getTrain()
x_test, y_test = Dataloader().getTest()
y_train = to_categorical(y_train)
x_train.pop("start_time")
x_train.pop("end_time")
x_test.pop("start_time")
x_test.pop("end_time")
print(x_train.shape)
print(y_train.shape)
clf = RandomForestClassifier(n_estimators=1000, random_state=0, n_jobs=-1)
# Apply The Full Featured Classifier To The Test Data
clf.fit(x_train, y_train)
sel = SelectFromModel(clf)
sel.fit(x_train, y_train)
selected_feat = x_train.columns[(sel.get_support())]
len(selected_feat)
print(selected_feat)
def getSelectedFeature():