def train(file):
y_data=makeRawDataset(file) #provides with raw data
X=pd.read_csv(file).Question
X_without=removePunc(X) #without punctuations
X_correct=fuzzy(X_without) #with fuzzy
X_enc=encode(X_correct)
labels=[">","<","<=",">=","==","NULL","LIKE"]
encoder=LabelEncoder()
codes=encoder.fit_transform(labels)
codeMap={labels[i]:codes[i] for i in range(len(labels))}
inverseMap={codes[i]:labels[i] for i in range(len(labels))}
maps={"codeMap":codeMap,"inverseMap":inverseMap}
np.save("Map.npy",maps)
y=[]
for i in y_data:
y.append(codeMap[i])
X_train, X_test, y_train, y_test = train_test_split(X_enc, y, test_size = 0.2, random_state = 42)
model = XGBClassifier()
model.fit(X_train,y_train)
model.save_model("WhereCond.model")
y_hat=model.predict(X_test)
print(y_hat[:10])
y_pred=[]
for i in y_hat:
y_pred.append(inverseMap[i])
y_true=[]
for i in y_test:
y_true.append(inverseMap[i])
sk_report = classification_report(digits=6,y_true=y_test,y_pred=y_hat)
print(sk_report)
def test(x,modelFile):
model = Booster() #init model
model.load_model(modelFile) # load data
maps=np.load("Map.npy",allow_pickle=True)
x_enc=encode([x])
y_enc=model.predict(DMatrix(x_enc))
y_pred=np.argmax(y_enc)
inverseMap=maps.item().get("inverseMap")
y_hat=inverseMap[y_pred]
print(y_hat)
def main(args):
data = json.load(open(args.input_refexps_json, 'r'))
max_length = 0
all_refexps = []
for keys in data:
for ref_id in data[keys]:
all_refexps.append(data[keys][ref_id])
for r in all_refexps:
t = tokenize(
r,
punct_to_keep=[',', ';'],
punct_to_remove=['?', '.']
)
if len(t) > max_length:
max_length = len(t)
refexp_token_to_idx = build_vocab(
all_refexps,
punct_to_keep=[',', ';'],
punct_to_remove=['?', '.']
)
with open(args.output_vocab_json, 'w') as f:
json.dump(refexp_token_to_idx, f)
with h5py.File(args.output_refexps_h5df, 'w') as f:
for keys in data:
one_image_refexps = []
# img_name = keys.split('.')[0]
one_image_refexps_to_idx = []
img_all_refexps = data[keys]
for ref_id in img_all_refexps:
# refexp = img_all_refexps[ref_id]
# one_image_refexps.append(refexp)
refexp = img_all_refexps[ref_id]
one_image_refexps.append(refexp)
for refexps in one_image_refexps:
tokens = tokenize(refexps, punct_to_remove=['?', '.'], punct_to_keep=[';', ','])
refexps_idx = encode(tokens, refexp_token_to_idx)
one_image_refexps_to_idx.append(refexps_idx)
for refexp_ in one_image_refexps_to_idx:
num_null = max_length - len(refexp_)
if num_null > 0:
refexp_ += [refexp_token_to_idx['<NULL>']]*num_null
one_image_refexps_to_idx_numpy = np.asarray(one_image_refexps_to_idx, dtype=np.int32)
f.create_dataset(keys, data=one_image_refexps_to_idx_numpy)
def predict():
if request.method == "GET":
return render_template("index.html")
else:
document = request.form["document"]
document = document.strip()
inp = []
inp.append(preprocess.encode(document))
inp = np.array(inp)
y_pred = model.predict(inp)
out = preprocess.decode(y_pred[-1])
return render_template("index.html", message=out, document=document)
def predict():
# get the car details from the form
car_details = list(request.form.values())
# preprocess the car details
car_details_processed = preprocess.encode(car_details)
# apply scaler to the car details
final_car_details = preprocess.scale(car_details_processed)
# predict the car price
car_price = model.predict(final_car_details)
# round the price value
car_price = round(car_price[0])
return render_template('index.html', prediction_text=f'The price of the {car_details[0].capitalize()} {car_details[1]} {car_details[3]} model grade {car_details[5]} {car_details[4]}, {car_details[6]} car with mileage value of {car_details[2]}km should be {car_price:,} naira')
res[ind] = 0.025641
result.write(str(index)+',')
result.write(",".join(map(str,res)))
result.write('\n')
result.close()
dir = os.getcwd()
train = pd.read_csv(dir+'/train.csv',sep = ',',header = 0)
train = train.sample(n = 50000)
train = pre.preprocess(train)
train = pre.encode(train)
print "Preprocessing done"
test = pd.read_csv(dir+'/test.csv',sep = ',',header = 0)
test = pre.preprocess(test)
test = pre.encode(test,True)
input_cols= train.columns[range(2,3)+range(4,7)+range(8,21)]
print input_cols
train_inp = train[input_cols]
train_op = train['Category']
test = test[input_cols]
for ind in range(len(res)):
if str(res[ind]) == 'nan':
res[ind] = 0.025641
result.write(str(index) + ',')
result.write(",".join(map(str, res)))
result.write('\n')
result.close()
dir = os.getcwd()
train = pd.read_csv(dir + '/train.csv', sep=',', header=0)
train = train.sample(n=50000)
train = pre.preprocess(train)
train = pre.encode(train)
print "Preprocessing done"
test = pd.read_csv(dir + '/test.csv', sep=',', header=0)
test = pre.preprocess(test)
test = pre.encode(test, True)
input_cols = train.columns[range(2, 3) + range(4, 7) + range(8, 21)]
print input_cols
train_inp = train[input_cols]
train_op = train['Category']
test = test[input_cols]
df_airport_test]).reset_index(drop=True)
print('Data Imported')
# Calculates the taxi out time
df_airport = preprocess.calc_TO_time(df_airport)
# Adds distance between stand and runway
df_airport = merging.merge_distance(df_airport)
# Calculates traffic feature
df_airport = merging.merge_traffic(df_airport)
# Calculates Q (queue size)
df_airport = merging.get_Q(df_airport)
print('Feature engineering in process')
# Merges weather features
df_airport = merging.get_weather_data(df_airport)
# Calculates the last known taxi times for the aircrafts
df_airport = preprocess.get_previous_taxi_times(df_airport, 3)
print('Feature engineering still in process')
# Calculates the last known taxi times by runway
df_airport = preprocess.get_previous_taxi_times_by_runway(df_airport, 3)
# Seperates the datetime columns into its attributes
df_airport = preprocess.get_date_attributes(df_airport)
# Converts datetime variables into cyclic features
df_airport = preprocess.encode(df_airport, 'aobt_hour', 23)
df_airport = preprocess.encode(df_airport, 'aobt_month', 12)
df_airport = preprocess.encode(df_airport, 'aobt_day', 365)
# Merge Aircraft characteristic features
df_airport = merging.merge_tech(df_airport)
# Outputs the full dataset with all features to be used for modelling
df_airport.to_csv('df_preprocessed_2015-2019.csv', index=False)
print('File created')
result.write(str(index)+',')
result.write(",".join(map(str,res)))
result.write('\n')
result.close()
train = pd.read_csv(dir+'/train.csv',sep = ',',header = 0)
train = train.sample(n = 450000)
validate = pd.read_csv(dir+'/train.csv',sep = ',',header = 0,nrows = 10000,skiprows = range(1,500000))
test = pd.read_csv(dir+'/test.csv',sep = ',',header = 0)
train = pre.preprocess(train)
train = pre.encode(train)
print "Training data preprocessed"
validate = pre.preprocess(validate)
validate = pre.encode(validate)
test= pre.preprocess(test)
test = pre.encode(test,True)
print "Test data preprocessed"
input_cols= train.columns[range(2,3)+range(4,7)+range(8,21)]
print input_cols
train_inp = train[input_cols]
na_values=missing_values)
predData = pd.read_csv("./Data/prediction-test/test.csv",
na_values=missing_values)
#import pdb; pdb.set_trace()
pd.set_option('display.max_rows', len(dataset))
pd.set_option('display.max_columns', None)
#Clean and encode varibales
dataset = pp.cleanData(dataset)
predData = pp.cleanData(predData)
encoder = preprocessing.LabelEncoder()
#dataset = pp.removeOutliers(dataset)
dataset, predData = pp.encode(dataset, predData, encoder)
#SPLITTING
train, test = train_test_split(dataset, test_size=0.2, random_state=42)
train = pp.removeOutliers(train)
cols = [
col for col in train.columns if col not in ['Instance', 'Income in EUR']
]
X_train = train[cols]
cols = [
col for col in test.columns if col not in ['Instance', 'Income in EUR']
]
X_test = test[cols]
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
import numpy as np
from sklearn.model_selection import train_test_split, KFold, cross_val_score
from preprocess import fillValues,encode,encode_test
import pickle as pkl
N_DATA = 8339
def diff(li1, li2):
return (list(set(li1) - set(li2)))
df = pd.read_csv('newdata.csv')
params_to_use = ['Binding', 'Brand', 'ListPrice', 'ProductGroup', 'location', 'govSchemeSize', 'companiesMoving', 'supplyDelhi', 'supplyMumbai', 'supplyKolkata', 'supplyChennai', 'demandDelhi', 'demandMumbai', 'demandChennai', 'demandKolkata', 'label']
data = df[params_to_use]
#print(data.info())
#fill Nan values
data = fillValues(data)
#encode strings
data = encode(data)
print(data.loc[8])
X = data[diff(params_to_use, ['label'])].as_matrix()
y = data['label'].as_matrix()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15, random_state=7)
dir = os.getcwd() ''' https://github.com/tqchen/xgboost/blob/master/demo/multiclass_classification/train.py ''' categories = ['WARRANTS', 'OTHER OFFENSES', 'LARCENY/THEFT', 'VEHICLE THEFT', 'VANDALISM', 'NON-CRIMINAL', 'ROBBERY', 'ASSAULT', 'WEAPON LAWS', 'BURGLARY', 'SUSPICIOUS OCC', 'DRUNKENNESS', 'FORGERY/COUNTERFEITING', 'DRUG/NARCOTIC', 'STOLEN PROPERTY', 'SECONDARY CODES', 'TRESPASS', 'MISSING PERSON', 'FRAUD', 'KIDNAPPING', 'RUNAWAY', 'DRIVING UNDER THE INFLUENCE', 'SEX OFFENSES FORCIBLE', 'PROSTITUTION', 'DISORDERLY CONDUCT', 'ARSON', 'FAMILY OFFENSES', 'LIQUOR LAWS', 'BRIBERY', 'EMBEZZLEMENT', 'SUICIDE', 'LOITERING', 'SEX OFFENSES NON FORCIBLE', 'EXTORTION', 'GAMBLING', 'BAD CHECKS', 'TREA', 'RECOVERED VEHICLE', 'PORNOGRAPHY/OBSCENE MAT'] train = pd.read_csv(dir+'/train.csv',sep = ',',header = 0,nrows = 100000) #train = train.sample(n = 450000) validate = pd.read_csv(dir+'/train.csv',sep = ',',header = 0,nrows = 10000,skiprows = range(1,300000)) test = pd.read_csv(dir+'/test.csv',sep = ',',header = 0,nrows = 5000) train = pre.preprocess(train) train = pre.encode(train) print "Training data preprocessed" validate = pre.preprocess(validate) validate = pre.encode(validate) test= pre.preprocess(test) test = pre.encode(test,True) print "Test data preprocessed" input_cols= train.columns[range(2,3)+range(4,7)+range(8,21)] print input_cols train_inp = train[input_cols].values train_op = train['Category'].values validate_inp = validate[input_cols].values
def main(args):
if (args.input_vocab_json == '') and (args.output_vocab_json == ''):
print("Must give one of --input_vocab_json or --output_vocab_json")
return
print("Loading questions...")
with open(args.input_questions, 'r') as f:
questions = f.read()
questions = questions.split("\n")
questions = questions[:-1]
print("Loading answers...")
with open(args.input_answers, 'r') as f:
answers = f.read()
answers = answers.split("\n")
answers = answers[:-1]
answer_token_to_idx = None
# Either create the vocab or load it from disk
if args.input_vocab_json == "" or args.expand_vocab == 1:
print("Building vocab...")
# Convert the answer tokens to unique id
answer_token_to_idx = build_vocab([answer for answer in answers])
# convert the tokens in all questions to unique id
question_token_to_idx = build_vocab(
[question for question in questions],
min_token_count=args.unk_threshold,
punct_to_keep=[';', ','],
punct_to_remove=['?', '.'])
answer_idx_to_token = {}
question_idx_to_token = {}
# create a reverse dictionary for answer idx to token mapping
for key, value in answer_token_to_idx.items():
answer_idx_to_token[value] = key
# create a reverse dictionary for question idx to token mapping
for key, value in question_token_to_idx.items():
question_idx_to_token[value] = key
# dump all the dictionaries as a single JSON file
vocab = {
"question_token_to_idx": question_token_to_idx,
"answer_token_to_idx": answer_token_to_idx,
"question_idx_to_token": question_idx_to_token,
"answer_idx_to_token": answer_idx_to_token
}
if args.input_vocab_json != "":
print("Loading vocab...")
if args.expand_vocab == 1:
new_vocab = vocab
with open(args.input_vocab_json, 'r') as f:
vocab = json.load(f)
if args.expand_vocab == 1:
num_new_words = 0
for word in new_vocab["question_token_to_idx"]:
if word not in vocab["question_token_to_idx"]:
print("Found new word %s" % word)
idx = len(vocab["question_token_to_idx"])
vocab["question_token_to_idx"][word] = idx
num_new_words += 1
print("Found %d new words" % num_new_words)
if args.output_vocab_json != "":
with open(args.output_vocab_json, 'w') as f:
json.dump(vocab, f)
# Encode all questions and programs
# This converts question strings to integers
print("Encoding data")
questions_encoded = []
_answers = []
for question, answer in zip(questions, answers):
question_tokens = tokenize(question,
punct_to_keep=[';', ','],
punct_to_remove=['?', '.'])
question_encoded = encode(question_tokens,
vocab["question_token_to_idx"],
allow_unk=args.encode_unk == 1)
questions_encoded.append(question_encoded)
_answers.append(vocab["answer_token_to_idx"][answer])
# Pad encoded questions and programs
max_question_length = max(len(x) for x in questions_encoded)
for qe in questions_encoded:
while len(qe) < max_question_length:
qe.append(vocab["question_token_to_idx"]["<NULL>"])
# Create h5 dataset file
print("Writing output")
questions_encoded = np.asarray(questions_encoded, dtype=np.int32)
print("Questions encoded shape is {}".format(questions_encoded.shape))
print("Length of answer tokens is {}".format(len(answer_token_to_idx)))
with h5py.File(args.output_h5_file, 'w') as f:
f.create_dataset("questions", data=questions_encoded)
if len(_answers) > 0:
f.create_dataset("answers", data=np.asarray(_answers))
def main(args):
if (args.input_vocab_json == '') and (args.output_vocab_json == ''):
print('Must give one of --input_vocab_json or --output_vocab_json')
return
print('Loading data...')
with open(args.input_questions_json, 'r') as f:
questions = json.load(f)['questions']
'''Either create the vocab or load it from disk'''
if args.input_vocab_json == '' or args.expand_vocab == 1:
print('Building vocab...')
if 'answer' in questions[0]:
answer_token_to_idx = build_vocab((str(q['answer'])
for q in questions))
question_token_to_idx = build_vocab(
[q['question'] for q in questions],
min_token_count=args.unk_threshold,
punct_to_keep=[';', ','],
punct_to_remove=['?', '.']
)
all_program_strs = []
for q in questions:
if 'program' not in q.keys():
continue
program_str = program_to_str(q['program'], args.mode)
if program_str is not None:
all_program_strs.append(program_str)
program_token_to_idx = build_vocab(all_program_strs)
vocab = {
'question_token_to_idx': question_token_to_idx,
'program_token_to_idx': program_token_to_idx,
'answer_token_to_idx': answer_token_to_idx,
}
if args.input_vocab_json != '':
print('Loading vocab...')
if args.expand_vocab == 1:
new_vocab = vocab
with open(args.input_vocab_json, 'r') as f:
vocab = json.load(f)
if args.expand_vocab == 1:
num_new_words = 0
for word in new_vocab['question_token_to_idx']:
if word not in vocab['question_token_to_idx']:
print('Found new word %s' % word)
idx = len(vocab['question_token_to_idx'])
vocab['question_token_to_idx'][word] = idx
num_new_words += 1
print('Found %d new words' % num_new_words)
if args.output_vocab_json != '':
with open(args.output_vocab_json, 'w') as f:
json.dump(vocab, f)
'''Encode all questions and programs'''
print('Encoding data')
questions_encoded = []
programs_encoded = []
question_families = []
orig_idxs = []
image_idxs = []
answers = []
for orig_idx, q in enumerate(questions):
question = q['question']
orig_idxs.append(orig_idx)
image_idxs.append(q['image_index'])
if 'question_family_index' in q:
question_families.append(q['question_family_index'])
question_tokens = tokenize(question, punct_to_keep=[';', ','],
punct_to_remove=['?', '.'])
question_encoded = encode(question_tokens,
vocab['question_token_to_idx'],
allow_unk=args.encode_unk == 1)
questions_encoded.append(question_encoded)
if 'program' in q:
program = q['program']
program_str = program_to_str(program, args.mode)
program_tokens = tokenize(program_str)
program_encoded = encode(program_tokens,
vocab['program_token_to_idx'])
programs_encoded.append(program_encoded)
if 'answer' in q:
answers.append(vocab['answer_token_to_idx'][str(q['answer'])])
'''Pad encoded questions and programs'''
max_question_length = max(len(x) for x in questions_encoded)
for qe in questions_encoded:
while len(qe) < max_question_length:
qe.append(vocab['question_token_to_idx']['<NULL>'])
if len(programs_encoded) > 0:
max_program_length = max(len(x) for x in programs_encoded)
for pe in programs_encoded:
while len(pe) < max_program_length:
pe.append(vocab['program_token_to_idx']['<NULL>'])
'''Create h5 file'''
print('Writing output')
questions_encoded = np.asarray(questions_encoded, dtype=np.int32)
programs_encoded = np.asarray(programs_encoded, dtype=np.int32)
print('Questions encoded shape is {}'.format(questions_encoded.shape))
print('Programs encoded shape is {}'.format(programs_encoded.shape))
with h5py.File(args.output_h5_file, 'w') as f:
f.create_dataset('questions', data=questions_encoded)
f.create_dataset('image_idxs', data=np.asarray(image_idxs))
f.create_dataset('orig_idxs', data=np.asarray(orig_idxs))
if len(programs_encoded) > 0:
f.create_dataset('programs', data=programs_encoded)
if len(question_families) > 0:
f.create_dataset('question_families',
data=np.asarray(question_families))
if len(answers) > 0:
f.create_dataset('answers', data=np.asarray(answers))
dataInfo.general(X)
dataInfo.missing_value_per_column(X)
categorical, numerical = dataInfo.colType(X)
print("categorical columns: ",categorical)
print(X.sex.unique()) # nominal data --> one hot encode
print(X.smoker.unique()) # ordinal data --> ordinal encode
print(X.region.unique()) # nominal data --> one hot encode
X_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size=0.8, test_size=0.2,random_state=0)
preprocessor = preprocess.encode()
trans=preprocessor.fit_transform(X_train) #####################################new dataframe
plot.regplot(X_train, y_train)
'''
###for plotting
print(trans)
df = pd.DataFrame(trans, #mit ohe codierten...
columns=['age', 'sex', 'bmi', 'children', 'smoker', 'region']) ##recreate dataframe
plot.pairplot(df,y)
#####
'''
#linear Regression
###################################################################