# -*- coding: utf-8 -*-

"""zeyu-copy-jianghao.ipynb

Automatically generated by Colaboratory.

Original file is located at

https://colab.research.google.com/drive/1Btw6usC5aRY_f-71QhvkmYxnoRAQG7P3

"""

import os

import cv2

import pydicom

import pandas as pd

import numpy as np

import tensorflow as tf

import matplotlib.pyplot as plt

import random

from tqdm.notebook import tqdm

from sklearn.model_selection import train_test_split, KFold

from sklearn.metrics import mean_absolute_error

from tensorflow_addons.optimizers import RectifiedAdam

from tensorflow.keras import Model

import tensorflow.keras.backend as K

import tensorflow.keras.layers as L

import tensorflow.keras.models as M

from tensorflow.keras.optimizers import Nadam

import seaborn as sns

import plotly.express as px

import plotly.graph_objects as go

from PIL import Image

def seed_everything(seed=2020):

tf.random.set_seed(seed)

seed_everything(42)

config = tf.compat.v1.ConfigProto()

config.gpu_options.allow_growth = True

session = tf.compat.v1.Session(config=config)

"""## 2. My upgrade <a class="anchor" id="2"></a>

[Back to Table of Contents](#0.1)

## 2.1. Commit now <a class="anchor" id="2.1"></a>

[Back to Table of Contents](#0.1)

"""

PATH="osic-pulmonary-fibrosis-progression/"

Dropout_model = 0.385

FVC_weight = 0.2

Confidence_weight = 0.2

commits_df = pd.DataFrame(columns = ['commit_num', 'FVC_weight', 'Dropout_model', 'LB_score', 'seed'])

"""Other parameters should be changed to increase accuracy.

## 3. Download data, auxiliary functions and model tuning <a class="anchor" id="3"></a>

[Back to Table of Contents](#0.1)

"""

train = pd.read_csv(PATH+'train.csv')

def get_tab(df):

return np.array(vector)

A = {}

TAB = {}

P = []

for i, p in tqdm(enumerate(train.Patient.unique())):

sub = train.loc[train.Patient == p, :]

fvc = sub.FVC.values

weeks = sub.Weeks.values

c = np.vstack([weeks, np.ones(len(weeks))]).T

a, b = np.linalg.lstsq(c, fvc)[0]

A[p] = a

TAB[p] = get_tab(sub)

P.append(p)

def get_img(path):

d = pydicom.dcmread(path)

return cv2.resize(d.pixel_array / 2**11, (512, 512))

from tensorflow.keras.utils import Sequence

class IGenerator(Sequence):

return [x, tab] , a

import efficientnet.tfkeras as efn

import tensorflow.keras.applications as tfka

from tensorflow.keras.layers import (

Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,

LeakyReLU, Concatenate

)

import efficientnet.tfkeras as efn

def get_efficientnet(model, shape):

return models_dict[model]

def build_model(shape=(512, 512, 1), model_class=None):

return model

model_classes = ['b5'] #['b0','b1','b2','b3',b4','b5','b6','b7']

models = [build_model(shape=(512, 512, 1), model_class=m) for m in model_classes]

print('Number of models: ' + str(len(models)))

tr_p, vl_p = train_test_split(P, shuffle=True, train_size = 0.8)

def score(fvc_true, fvc_pred, sigma):

return np.mean(metric)

subs = []

for model in models:

metric = []

# for q in tqdm(range(1, 10)):

# m = []

# for p in vl_p:

# x = []

# tab = []

# if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:

# continue

# ldir = os.listdir(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/')

# for i in ldir:

# if int(i[:-4]) / len(ldir) < 0.8 and int(i[:-4]) / len(ldir) > 0.15:

# x.append(get_img(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))

# tab.append(get_tab(train.loc[train.Patient == p, :]))

# if len(x) < 1:

# continue

# tab = np.array(tab)

# x = np.expand_dims(x, axis=-1)

# _a = model.predict([x, tab])

# a = np.quantile(_a, q / 10)

# percent_true = train.Percent.values[train.Patient == p]

# fvc_true = train.FVC.values[train.Patient == p]

# weeks_true = train.Weeks.values[train.Patient == p]

# fvc = a * (weeks_true - weeks_true[0]) + fvc_true[0]

# percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])

# m.append(score(fvc_true, fvc, percent))

# print(np.mean(m))

# metric.append(np.mean(m))

# q = (np.argmin(metric) + 1)/ 10

q=0.8

sub = pd.read_csv(PATH+'sample_submission.csv')

test = pd.read_csv(PATH+'test.csv')

A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}

STD, WEEK = {}, {}

for p in test.Patient.unique():

x = []

tab = []

ldir = os.listdir(PATH+f'test/{p}/')

for i in ldir:

if int(i[:-4]) / len(ldir) < 0.8 and int(i[:-4]) / len(ldir) > 0.15:

x.append(get_img(PATH+f'test/{p}/{i}'))

tab.append(get_tab(test.loc[test.Patient == p, :]))

if len(x) <= 1:

continue

tab = np.array(tab)

x = np.expand_dims(x, axis=-1)

_a = model.predict([x, tab])

a = np.quantile(_a, q)

A_test[p] = a

B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]

P_test[p] = test.Percent.values[test.Patient == p]

WEEK[p] = test.Weeks.values[test.Patient == p]

for k in sub.Patient_Week.values:

p, w = k.split('_')

w = int(w)

fvc = A_test[p] * w + B_test[p]

sub.loc[sub.Patient_Week == k, 'FVC'] = fvc

sub.loc[sub.Patient_Week == k, 'Confidence'] = (

P_test[p] - A_test[p] * abs(WEEK[p] - w)

)

_sub = sub[["Patient_Week","FVC","Confidence"]].copy()

subs.append(_sub)

"""## 4. Prediction and submission <a class="anchor" id="4"></a>

[Back to Table of Contents](#0.1)

## 4.1 Average prediction <a class="anchor" id="4.1"></a>

[Back to Table of Contents](#0.1)

"""

N = len(subs)

sub = subs[0].copy() # ref

sub["FVC"] = 0

sub["Confidence"] = 0

for i in range(N):

sub["FVC"] += subs[0]["FVC"] * (1/N)

sub["Confidence"] += subs[0]["Confidence"] * (1/N)

sub.head()

sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False)

img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()

"""## 4.2 Osic-Multiple-Quantile-Regression <a class="anchor" id="4.2"></a>

[Back to Table of Contents](#0.1)

"""

ROOT = PATH

BATCH_SIZE=128

tr = pd.read_csv(f"{ROOT}/train.csv")

tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])

chunk = pd.read_csv(f"{ROOT}/test.csv")

print("add infos")

sub = pd.read_csv(f"{ROOT}/sample_submission.csv")

sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])

sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))

sub = sub[['Patient','Weeks','Confidence','Patient_Week']]

sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")

tr['WHERE'] = 'train'

chunk['WHERE'] = 'val'

sub['WHERE'] = 'test'

data = tr.append([chunk, sub])

print(tr.shape, chunk.shape, sub.shape, data.shape)

print(tr.Patient.nunique(), chunk.Patient.nunique(), sub.Patient.nunique(),

data.Patient.nunique())

data['min_week'] = data['Weeks']

data.loc[data.WHERE=='test','min_week'] = np.nan

data['min_week'] = data.groupby('Patient')['min_week'].transform('min')

base = data.loc[data.Weeks == data.min_week]

base = base[['Patient','FVC']].copy()

base.columns = ['Patient','min_FVC']

base['nb'] = 1

base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')

base = base[base.nb==1]

base.drop('nb', axis=1, inplace=True)

data = data.merge(base, on='Patient', how='left')

data['base_week'] = data['Weeks'] - data['min_week']

del base

COLS = ['Sex','SmokingStatus'] #,'Age'

FE = []

for col in COLS:

for mod in data[col].unique():

FE.append(mod)

data[mod] = (data[col] == mod).astype(int)

#

data['age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )

data['BASE'] = (data['min_FVC'] - data['min_FVC'].min() ) / ( data['min_FVC'].max() - data['min_FVC'].min() )

data['week'] = (data['base_week'] - data['base_week'].min() ) / ( data['base_week'].max() - data['base_week'].min() )

data['percent'] = (data['Percent'] - data['Percent'].min() ) / ( data['Percent'].max() - data['Percent'].min() )

FE += ['age','percent','week','BASE']

tr = data.loc[data.WHERE=='train']

chunk = data.loc[data.WHERE=='val']

sub = data.loc[data.WHERE=='test']

del data

tr.shape, chunk.shape, sub.shape

"""## 4.3 The change of mloss <a class="anchor" id="4.3"></a>

[Back to Table of Contents](#0.1)

"""

C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")

def score(y_true, y_pred):

return K.mean(metric)

def qloss(y_true, y_pred):

return K.mean(v)

def mloss(_lambda):

def loss(y_true, y_pred):

return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)

return loss

def make_model(nh):

return model

y = tr['FVC'].values

z = tr[FE].values

ze = sub[FE].values

nh = z.shape[1]

pe = np.zeros((ze.shape[0], 3))

pred = np.zeros((z.shape[0], 3))

net = make_model(nh)

print(net.summary())

print(net.count_params())

NFOLD = 5 # originally 5

kf = KFold(n_splits=NFOLD)

cnt = 0

EPOCHS = 855

# for tr_idx, val_idx in kf.split(z):

# cnt += 1

# print(f"FOLD {cnt}")

# net = make_model(nh)

# net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,

# validation_data=(z[val_idx], y[val_idx]), verbose=0) #

# print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))

# print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))

# print("predict val...")

# net.save(str(cnt)+'regression_weight.h5')

# pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)

# print("predict test...")

# pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0) / NFOLD

for tr_idx, val_idx in kf.split(z):

cnt += 1

print(f"FOLD {cnt}")

net = make_model(nh)

net.load_weights('regression_weight.h5')

print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))

print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))

print("predict val...")

pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)

print("predict test...")

pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)

break

sigma_opt = mean_absolute_error(y, pred[:, 1])

unc = pred[:,2] - pred[:, 0]

sigma_mean = np.mean(unc)

print(sigma_opt, sigma_mean)

idxs = np.random.randint(0, y.shape[0], 100)

plt.plot(y[idxs], label="ground truth")

plt.plot(pred[idxs, 0], label="q25")

plt.plot(pred[idxs, 1], label="q50")

plt.plot(pred[idxs, 2], label="q75")

plt.legend(loc="best")

# plt.show()

print(unc.min(), unc.mean(), unc.max(), (unc>=0).mean())

plt.hist(unc)

plt.title("uncertainty in prediction")

# plt.show()

sub.head()

# PREDICTION

sub['FVC1'] = 1.*pe[:, 1]

sub['Confidence1'] = pe[:, 2] - pe[:, 0]

subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()

subm.loc[~subm.FVC1.isnull()].head(10)

subm.loc[~subm.FVC1.isnull(),'FVC'] = subm.loc[~subm.FVC1.isnull(),'FVC1']

if sigma_mean<70:

subm['Confidence'] = sigma_opt

else:

subm.loc[~subm.FVC1.isnull(),'Confidence'] = subm.loc[~subm.FVC1.isnull(),'Confidence1']

subm.head()

subm.describe().T

otest = pd.read_csv(PATH+'test.csv')

for i in range(len(otest)):

subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]

subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1

subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False)

reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()

"""## 4.4 Ensemble and blending <a class="anchor" id="4.4"></a>

[Back to Table of Contents](#0.1)

"""

df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True).reset_index(drop=True)

df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True).reset_index(drop=True)

df = df1[['Patient_Week']].copy()

df['FVC'] = FVC_weight*df1['FVC'] + (1-FVC_weight)*df2['FVC']

df['Confidence'] = 0.05*df1['Confidence'] + (0.95)*df2['Confidence']

df.head()

df.to_csv('submission.csv', index=False)

print(FVC_weight,Confidence_weight)

# -*- coding: utf-8 -*-

"""fork-of-feature-engineering-with-a-linear-model-1a.ipynb

Automatically generated by Colaboratory.

Original file is located at

https://colab.research.google.com/drive/1Hsnybp6Cyqi1MUYjpf3EbIKrBhkdo5rX

When I first saw the osic pulmonary fibrosis competition I thought it looked like quite an interesting competition for its mix of tabular and image data and trying to produce a forecast based on that. This provides scope to try all sorts of things such as feature engineering, image embedding models and time series models. To get started I have focused on the tabular data only which as it turns out can get us quite far before even touching the images. I've tried to stay away from spending too much time on model selection and instead opted for feature experimentation.

But first a quick overview (disclaimer I'm not a medical expert so apologies if I get any of the medical stuff wrong). Pulmonary Fibrosis is a lung disease that causes the lungs to decline over time. The rate of decline can range between very rapid and very slow. We can measure the diseases progress through [FVC](https://lunginstitute.com/blog/fev1-and-fvc/#:~:text=The%20forced%20vital%20capacity%20(FVC,the%20severity%20of%20the%20condition)) (Forced Vital Capacity) which involves the patient taking a deep breath and blowing as hard as they can into a tube. The amount of air they blow out is the FVC measured in ml.

So the challenge is when given a starting FVC for a patient to forecast the decline in the FVC value for all the upcoming weeks.

"""

print("------------------1")

import numpy as np

import pandas as pd

import matplotlib.pyplot as plt

from sklearn import linear_model, ensemble

from sklearn.metrics import mean_squared_error, mean_absolute_error

import tensorflow as tf

from tqdm.notebook import tqdm

import os

from PIL import Image

"""## Load data

Let's begin by loading the data.

"""

train = pd.read_csv(PATH+'train.csv')

test = pd.read_csv(PATH+'test.csv')

submission = pd.read_csv(PATH+'sample_submission.csv')

train.head()

train.info()

test.head()

test.info()

"""So there's not a huge volume of tabular data. 1.5k of training examples with only 5 columns (weeks, percent, age, sex, smoking status) to construct features from.

## Merge datasets

I've learned recently that it is good practice to merge the train, validation and test sets at the start of a notebook. This ensures that exactly the same transformation is applied to every example. First though, I'll make sure there are no duplicates in the training dataset.

"""

train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])

"""Then form the submission dataset. The test dataset needs expanding out across the 146 weeks per patient that the competition requires. This can be achieved by joining it to the sample submission."""

submission['Patient'] = (

submission['Patient_Week']

.apply(

lambda x:x.split('_')[0]

)

)

submission['Weeks'] = (

submission['Patient_Week']

.apply(

lambda x: int(x.split('_')[-1])

)

)

submission = submission[['Patient','Weeks', 'Confidence','Patient_Week']]

submission = submission.merge(test.drop('Weeks', axis=1), on="Patient")

submission.head()

"""Mark each example in each dataset with the name of the dataset they come from. This enables me to quickly split the dataset back up into the three component pieces at the end of the notebook."""

train['Dataset'] = 'train'

test['Dataset'] = 'test'

submission['Dataset'] = 'submission'

"""Merge the datasets into one and reset the index."""

all_data = train.append([test, submission])

all_data = all_data.reset_index()

all_data = all_data.drop(columns=['index'])

all_data.head()

"""## Quick data analysis

I think it's worth having a look at how the FVC (label) value declines for a sample of patients in the training dataset. Let's pick the first five in the data and plot the decline of the FVC.

"""

train_patients = train.Patient.unique()

fig, ax = plt.subplots(5, 1, figsize=(10, 20))

for i in range(5):

patient_log = train[train['Patient'] == train_patients[i]]

ax[i].set_title(train_patients[i])

ax[i].plot(patient_log['Weeks'], patient_log['FVC'])

"""So the decline is kinda linear as it does generally trend down over time. There are some spikes back up along the way which could cause a few issues. However this means that a simple linear model could have a good go at producing forecasts on this challenge as the main thing we need to do is predict the rate of decline for a patient. Like a trend line for these charts.

## Feature Engineering

There is some good scope for engineering new features for this model.

### First FVC and First Week

Some useful features might be the first FVC recorded per patient and the week it was recorded in

"""

all_data['FirstWeek'] = all_data['Weeks']

all_data.loc[all_data.Dataset=='submission','FirstWeek'] = np.nan

all_data['FirstWeek'] = all_data.groupby('Patient')['FirstWeek'].transform('min')

first_fvc = (

)

all_data = all_data.merge(first_fvc, on='Patient', how='left')

all_data.head()

"""### Weeks Passed

This feature measures how many weeks have passed since the patients first FVC reading.

"""

all_data['WeeksPassed'] = all_data['Weeks'] - all_data['FirstWeek']

all_data.head()

"""### Patient height

Apparently the height of a patient is an important variable when predicting FVC. Maybe tall people have larger lungs and thus more air to exhale. Thanks to Srikanth Potukuchi whose [notebook](https://www.kaggle.com/srikanthpotukuchi/osic-random-forest-new-height-extracted) showed me how to estimate a patients height using their FVC.

"""

def calculate_height(row):

return row['FirstFVC'] / (21.78 - 0.101 * row['Age'])

all_data['Height'] = all_data.apply(calculate_height, axis=1)

all_data.head()

"""### Categorical columns

The sex and smoking status columns are categorical columns that need some transformation to turn them into numbers. Pandas get dummies makes this easy to achieve.

"""

all_data = pd.concat([

], axis=1)

all_data = all_data.drop(columns=['Sex', 'SmokingStatus'])

# print(all_data.Age.max())

pd.value_counts(all_data.Age)

age_divide = pd.cut(all_data.Age,[49,65,70,79,88],labels=['65','70','75','88'])

pd.value_counts(age_divide)

all_data['AgeGroup'] = age_divide

all_data=all_data.join(pd.get_dummies(all_data.AgeGroup))

all_data = all_data.drop(columns=['Age','AgeGroup'])

all_data.head()

# print(all_data.Height.max())

# print(all_data.Height.median())

# pd.set_option('display.max_rows',None)

# print(all_data.Height.sort_values())

''''''''''

print(all_data.Height.max())

pd.value_counts(all_data.Height)

Height_divide = pd.cut(all_data.Height,[69.000680,122.094692,145.5241080483083,75,325.185486],labels=['Height_65','Height_70','Height_75','Height_88'])

pd.value_counts(Height_divide)

all_data['HeightGroup'] = Height_divide

all_data=all_data.join(pd.get_dummies(all_data.HeightGroup))

all_data.head()

'''''''''

all_data.head()

"""### Scale features

Now scale all the features to get them onto the same range of numbers (0-1).

"""

def scale_feature(series):

return (series - series.min()) / (series.max() - series.min())

all_data['Weeks'] = scale_feature(all_data['Weeks'])

all_data['Percent'] = scale_feature(all_data['Percent'])

# all_data['Age'] = scale_feature(all_data['Age'])

all_data['FirstWeek'] = scale_feature(all_data['FirstWeek'])

all_data['FirstFVC'] = scale_feature(all_data['FirstFVC'])

all_data['WeeksPassed'] = scale_feature(all_data['WeeksPassed'])

all_data['Height'] = scale_feature(all_data['Height'])

"""Specify what columns will be used as features. This is for easy filtering of the datasets later."""

feature_columns = [

]

"""### Split dataframe

Split the data back into the three dataframes they started as.

"""

train = all_data.loc[all_data.Dataset == 'train']

test = all_data.loc[all_data.Dataset == 'test']

submission = all_data.loc[all_data.Dataset == 'submission']

"""And take a look at the features that will be used to train the model."""

train[feature_columns].head()

"""## Model

As I mentioned at the start of the notebook I didn't intend to experiment too much with model selection in this notebook so I have stuck with a simple Linear Regressor from SKLearn. While these non-deep models don't usually produce as accurate results as deep models they are super fast to train and easy to evaluate. This is great for notebooks focussing primarily on features.

"""

model = linear_model.HuberRegressor( max_iter=200)

"""With the model loaded, insert the features and labels for training."""

model.fit(train[feature_columns], train['FVC'])

"""Make predictions"""

predictions = model.predict(train[feature_columns])

"""## Evaluate

Let's begin by having a look at the models weights. This gives us a good indication of what features are driving the models predictions.

"""

plt.bar(train[feature_columns].columns.values, model.coef_)

plt.xticks(rotation=90)

# plt.show()

"""While mean squared error isn't the competition metric it is a simple loss metric to help understand how close the models predictions are to the actual labels. The limitation of this error number though is that it can't be too close to zero as that would indicate over-fitting a model that should only be producing a trend line."""

mse = mean_squared_error(

)

mae = mean_absolute_error(

train['FVC'],

predictions

)

print('MSE Loss: {0:.2f}'.format(mse))

print('MAE Loss: {0:.2f}'.format(mae))

"""Found code for competition metric [here](https://www.kaggle.com/titericz/tabular-simple-eda-linear-model#Calculate-competition-metric)"""

def competition_metric(trueFVC, predFVC, predSTD):

clipSTD = np.clip(predSTD, 70 , 9e9)

deltaFVC = np.clip(np.abs(trueFVC - predFVC), 0 , 1000)

return np.mean(-1 * (np.sqrt(2) * deltaFVC / clipSTD) - np.log(np.sqrt(2) * clipSTD))

print(

'Competition metric: ',

competition_metric(train['FVC'].values, predictions, 285)

)

"""Let's also include a scatterplot and histogram to see an overview of how close the predictions are to the labels."""

train['prediction'] = predictions

plt.scatter(predictions, train['FVC'])

plt.xlabel('predictions')

plt.ylabel('FVC (labels)')

# plt.show()

delta = predictions - train['FVC']

plt.hist(delta, bins=20)

# plt.show()

"""Finally take the first five patients as a sample and compare the true FVC readings against the models predicted FVC readings."""

fig, ax = plt.subplots(5, 1, figsize=(10, 20))

for i in range(5):

patient_log = train[train['Patient'] == train_patients[i]]

ax[i].set_title(train_patients[i])

ax[i].plot(patient_log['WeeksPassed'], patient_log['FVC'], label='truth')

ax[i].plot(patient_log['WeeksPassed'], patient_log['prediction'], label='prediction')

ax[i].legend()

"""## Submission

With the model trained we can use it to make predictions. As the submission dataframe went through the same transformation as the train dataset it is ready for the model to infer on.

"""

submission[feature_columns].head()

"""Add the predictions to the submission dataframe."""

sub_predictions = model.predict(submission[feature_columns])

submission['FVC'] = sub_predictions

"""Plot the forecasted FVC for five patients in the test set."""

test_patients = list(submission.Patient.unique())

fig, ax = plt.subplots(5, 1, figsize=(10, 20))

for i in range(1):

patient_log = submission[submission['Patient'] == test_patients[i]]

ax[i].set_title(test_patients[i])

ax[i].plot(patient_log['WeeksPassed'], patient_log['FVC'])

"""Drop the surplus columns not needed for submission and add a dummy confidence column."""

submission = submission[['Patient_Week', 'FVC']]

submission['Confidence'] = 285

"""And write the submission to file."""

submission.to_csv('result.csv', index=False)

submission.head()

submission = submission.sort_values(by=['Patient_Week'], ascending=True).reset_index(drop=True)

submission.head()

df2.head()

df['FVC'] = 0.19*submission['FVC'] + 0.81*df['FVC']

df['Confidence'] = 0.975*df['Confidence']

df.head()

df_final = df_final1 = df

# print(df_final1)

# print(df2)

submission.head()

# submission = submission.sort_values(by=['Patient_Week'], ascending=True).reset_index(drop=True)

# for i in range(len(submission['Confidence'].values)):

# if submission['Confidence'][i] < df_final['Confidence'][i]:

# df_final['Confidence'][i] = 0.8*df_final['Confidence'][i] + 0.2*submission['Confidence'][i]

# print(df_final)

test = pd.read_csv(PATH+'test.csv')

for i in range(len(test.values)):

week = test['Weeks'][i]

patient = test['Patient'][i]

FVC = test['FVC'][i]

mask = df_final['Patient_Week'] == str(patient+'_'+str(week))

pos = np.flatnonzero(mask)

FVC_pre = df_final['FVC'][int(pos)]

# print(pos)

# print(FVC)

# print(FVC_pre)

if (FVC_pre - FVC) > 100:

mask = df_final['Patient_Week'] == str(patient+'_'+'131')

pos = np.flatnonzero(mask)

df_final['FVC'][int(pos)] -= pow((FVC_pre - FVC),0.5)

df_final['Confidence'][int(pos)] -= abs(pow((FVC_pre - FVC),0.7))

print('131_week',df_final['FVC'][int(pos)])

mask = df_final['Patient_Week'] == str(patient+'_'+'132')

pos = np.flatnonzero(mask)

df_final['FVC'][int(pos)] -= pow((FVC_pre - FVC),0.5)

df_final['Confidence'][int(pos)] -= abs(pow((FVC_pre - FVC),0.7))

mask = df_final['Patient_Week'] == str(patient+'_'+'133')

pos = np.flatnonzero(mask)

df_final['FVC'][int(pos)] -= pow((FVC_pre - FVC),0.5)

df_final['Confidence'][int(pos)] -= abs(pow((FVC_pre - FVC),0.7))

elif (FVC - FVC_pre) > 100:

mask = df_final['Patient_Week'] == str(patient+'_'+'131')

pos = np.flatnonzero(mask)

df_final['FVC'][int(pos)] += pow((-FVC_pre + FVC),0.5)

df_final['Confidence'][int(pos)] -= abs(pow((FVC_pre - FVC),0.7))

print('131_week',df_final['FVC'][int(pos)])

mask = df_final['Patient_Week'] == str(patient+'_'+'132')

pos = np.flatnonzero(mask)

df_final['FVC'][int(pos)] += pow((-FVC_pre + FVC),0.5)

df_final['Confidence'][int(pos)] -= abs(pow((FVC_pre - FVC),0.7))

mask = df_final['Patient_Week'] == str(patient+'_'+'133')

pos = np.flatnonzero(mask)

df_final['FVC'][int(pos)] += pow((-FVC_pre + FVC),0.5)

df_final['Confidence'][int(pos)] -= abs(pow((FVC_pre - FVC),0.7))

df_final.to_csv('result.csv', index=False)