def apply_transformations(X=None, y=None, save=False):
config = get_config()
if X is None:
X = np.load("data/processed/X_2c.npy")
y = np.load("data/processed/y_2c.npy")
aug = create_augmenter()
samples_to_augment = config["preprocessing"]["augmented_ds_size"]
X_augmented = np.empty((samples_to_augment, 64, 64, 3))
y_ = []
i = 0
for index, item in enumerate(X_augmented):
rnd = np.random.choice(X.shape[0], 1)[0]
X_augmented[index, :, :, :] = X[rnd]
y_.append(y[rnd])
X_augmented = np.concatenate([X_augmented, X], axis=0)
y_ = np.concatenate([y_, y])
X_aug = aug.augment_images(X_augmented)
if save:
np.save("data/processed/X_a.npy", X_augmented)
np.save("data/processed/y_a.npy", y_)
return X_aug, y_
def load_callbacks(weights_path):
config = get_config()
cbacks_config = config["train"]["callbacks"]
metrics = Metrics()
model_checkpoint = ModelCheckpoint(weights_path, monitor='val_loss', save_best_only=True)
#cbacks = [metrics, model_checkpoint]
cbacks = [ model_checkpoint]
lr_reduce = cbacks_config["lr_reduce"]
if lr_reduce['apply']:
params = ['monitor', 'factor', 'patience', 'verbose', 'mode', 'min_delta', 'cooldown', 'min_lr']
lr_reduce_cb = _load_config_params(params, ReduceLROnPlateau,lr_reduce)
cbacks.append(lr_reduce_cb)
early_stopping = cbacks_config["early_stopping"]
if early_stopping["apply"]:
params = ['monitor', 'min_delta', 'patience', 'mode', 'verbose']
early_stopping_cb = _load_config_params(params, EarlyStopping, early_stopping)
cbacks.append(early_stopping_cb)
return cbacks
def apply_rescale(X):
"""Normalizes a given dataset.
Parameters
----------
X: A numpy array of type (n, w, h, c)
Returns
-------
X_: X after applying normalization
"""
config = get_config()
if config["preprocessing"]["rescale"]:
if config["preprocessing"]["rescale_method"] == "normalize":
X_ = normalize_meanstd(X)
elif config["preprocessing"]["rescale_method"] == "standartize":
X_ = normalize_01(X)
else:
print("{} is not a valid rescaling method".format(
config["preprocessing"]["rescale_method"]))
else:
X_ = X
return X_
def load_optimizer():
config = get_config()
opt_config = config["train"]["optimizers"]
opt_name = opt_config["use"]
if (opt_name == "adam"):
params_to_parse = [
"lr", "beta_1", "beta_2", "epsilon", "decay", "amsgrad"
]
opt = _load_config_params(params_to_parse, Adam, opt_config)
elif (opt_name == "sgd"):
params_to_parse = ["lr", "rho", "epsilon", "decay"]
opt = _load_config_params(params_to_parse, SGD, opt_config)
elif (opt_name == "rmsprop"):
params_to_parse = ["lr", "rho", "epsilon", "decay"]
opt = _load_config_params(params_to_parse, SGD, opt_config)
opt = _load_config_params(params_to_parse, RMSprop, opt_config)
elif (opt_name == "nadam"):
params_to_parse = ["lr", "beta_1", "beta_2", "epsilon"]
opt = _load_config_params(params_to_parse, Nadam, opt_config)
return opt
def architecture(self):
config = get_config()
config = config["train"]["optimizers"]
channels, height, weight = 3, 500, 500
# Input
#input_shape = (height, weight, 3)
img_input = Input(shape=(64, 64, 1))
output = CrfRnnLayer(
image_dims=(64, 64),
num_classes=1,
theta_alpha=config["crf_theta_alpha"], #3
theta_beta=config["crf_theta_beta"], #3
theta_gamma=config["crf_theta_gamma"], #3
num_iterations=config["crf_num_iterations"],
name='crfrnn')([img_input, img_input])
#
k = Flatten()(output)
predictions = Dense(1, activation='sigmoid')(k)
model = Model(img_input, predictions, name='crfrnn_net')
return model
def create_augmented_dataset(X=None, y=None, save=False, return_data=False):
config = get_config()
X_, y_ = apply_transformations(X, y)
if save:
np.save("data/processed/X_train.npy", X_train)
np.save("data/processed/y_train.npy", y_train)
np.save("data/processed/X_val.npy", X_val)
np.save("data/processed/y_val.npy", y_val)
if return_data:
return X_, y_
def log_model(model, c_backs, metrics_dic, config=None):
""" Description
:type model: Keras model
:param model:
:type c_backs: Keras call_back file
:param c_backs:
:type config: A dict containing the current parameters
:param config:
:raises:
:rtype:
"""
import datetime
if config is None:
from src.helper import get_config
config = get_config()
now = datetime.datetime.now()
log = dict()
log["datetime"] = {
'date': now.strftime("%Y-%m-%d %H:%M"),
'unix': now.timestamp()
}
log["metric"] = metrics_dic
hist = model.history.history
hist["epoch"] = model.history.epoch
for cb in c_backs:
if cb.__class__.__name__ == 'Metrics':
hist["val_auroc"] = cb.val_auroc
hist["val_precision"] = cb.val_precisions
hist["val_recall"] = cb.val_recalls
log["parameters"] = config
log["history"] = hist
import os
file_path = os.path.join('logs', now.strftime("log_%Y_%m_%d_%H_%M.json"))
with open(file_path, 'a+') as fp:
json.dump(log, fp, cls=NumpyEncoder)
def architecture(self):
config = get_config()
config = config["train"]["optimizers"]
channels, height, weight = 3, 500, 500
input_shape = (height, weight, 3)
base_model = ResNet50(include_top=False,
input_shape=(self.input_shape),
classes=1,
weights=None)
x = base_model.output
score2 = Conv2DTranspose(1, (12, 12), strides=2, name='score2')(x)
# Final up-sampling and cropping
upsample = Conv2DTranspose(1, (12, 12),
strides=4,
name='upsample',
use_bias=False)(score2)
#upsample = Cropping2D((12, 12))(upsample)
img_input = base_model.input
x = ZeroPadding2D(padding=(218, 218))(img_input)
output = CrfRnnLayer(
image_dims=(64, 64),
num_classes=1,
theta_alpha=config["crf_theta_alpha"], #3
theta_beta=config["crf_theta_beta"], #3
theta_gamma=config["crf_theta_gamma"], #3
num_iterations=config["crf_num_iterations"],
name='crfrnn')([upsample, img_input])
x = Flatten()(output)
x = Dense(1024, activation='relu')(x)
x = Dropout(.5)(x)
x = Dense(1024, activation='relu')(x)
predictions = Dense(1, activation='sigmoid')(x)
model = Model(inputs=base_model.input, outputs=predictions)
return model
def get_image_patch(img_array, coords, padding=None):
""" Description
:type img_array: numpy array
:param img_array: a image comprised of a numpy array
:type coords: list(i,j)
:param coords: The coordinates where the crop will be centered, of type (i,j), i being the columns and j the row
:type padding: int or list(x,y)
:param padding: The padding that will be around the center coordinates. If an int, it will create a square image. If a list x is the horizontal padding and y the vertical
:raises:
:rtype:
"""
if padding is None:
from src.helper import get_config
config = get_config()
padding = config["general"]["padding"]
i = coords[0]
j = coords[1]
k = coords[2]
if isinstance(padding, list):
h_padding = padding[0]
v_padding = padding[1]
else:
h_padding = padding
v_padding = padding
try:
X_ = img_array[j - v_padding:j + v_padding,
i - h_padding:i + h_padding, k]
except:
return None
else:
if (X_.shape == (2 * padding, 2 * padding)):
return np.asarray(X_).reshape((1, 2 * padding, 2 * padding, 1))
class BaseArchitecture():
weights_directory = "data/interim/weights/"
name = "Base Architecture"
flaten_layer = None
config = get_config()
input_shape = (2 * config["general"]["padding"],
2 * config["general"]["padding"],
config["general"]["channels"])
def __init__(self, load_weights=False):
self.id = self.name.lower().strip().replace(" ", "_")
self.file = "{}.h5".format(self.id)
self.weights_path = os.path.join(self.weights_directory, self.file)
self.load_weights = load_weights
def architecture(self):
''' This method returns the keras architecture of the model. It should be implemented by each sub class '''
raise NotImplementedError
class DatabaseConnect:
""" handle db """
CONFIG = get_config()
def __init__(self):
self.conn, self.cur = self.db_connect()
def db_connect(self):
""" returns connection and curser """
# Connect to database
conn = psycopg2.connect(
host=self.CONFIG['postgres']['db_host'],
database=self.CONFIG['postgres']['db_database'],
user=self.CONFIG['postgres']['db_user'],
password=self.CONFIG['postgres']['db_password'])
# Open a cursor to perform database operations
cur = conn.cursor()
return conn, cur
def db_execute(self, query):
""" run a query """
if isinstance(query, str):
self.cur.execute(query)
rows = self.cur.fetchall()
elif isinstance(query, tuple):
self.cur.execute(query[0], query[1])
rows = False
return rows
def db_close(self):
""" clean close the conn and curser """
self.conn.commit()
self.cur.close()
self.conn.close()
def architecture(self):
config = get_config()
config = config["train"]["optimizers"]
channels, height, weight = 3, 500, 500
# Input
input_shape = (height, weight, 3)
img_input = Input(shape=self.input_shape)
#img_input = Cropping2D((3,3))(img_input)
# Add plenty of zero padding
x = ZeroPadding2D(padding=(218, 218))(img_input)
# VGG-16 convolution block 1
x = Conv2D(64, (3, 3), activation='relu', padding='valid', name='conv1_1')(x)
x = Conv2D(64, (3, 3), activation='relu', padding='same', name='conv1_2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool1')(x)
# VGG-16 convolution block 2
x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv2_1')(x)
x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv2_2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool2', padding='same')(x)
# VGG-16 convolution block output3
x = Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_1')(x)
x = Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_2')(x)
x = Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool3', padding='same')(x)
pool3 = x
# VGG-16 convolution block 4
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_1')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_2')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool4', padding='same')(x)
pool4 = x
# VGG-16 convolution block 5
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_1')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_2')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool5', padding='same')(x)
# Fully-connected layers converted to convolution layers
x = Conv2D(128, (7, 7), activation='relu', padding='valid', name='fc6')(x)
x = Dropout(0.5)(x)
x = Conv2D(128, (1, 1), activation='relu', padding='valid', name='fc7')(x)
x = Dropout(0.5)(x)
x = Conv2D(21, (1, 1), padding='valid', name='score-fr')(x)
# Deconvolution
score2 = Conv2DTranspose(1, (4, 4), strides=2, name='score2')(x)
# Skip connections from pool4
score_pool4 = Conv2D(1, (1, 1), name='score-pool4')(pool4)
score_pool4c = Cropping2D((5, 5))(score_pool4)
score_fused = Add()([score2, score_pool4c])
score4 = Conv2DTranspose(1, (4, 4), strides=2, name='score4', use_bias=False)(score_fused)
# Skip connections from pool3
score_pool3 = Conv2D(1, (1, 1), name='score-pool3')(pool3)
score_pool3c = Cropping2D((9, 9))(score_pool3)
score_pool3c = ZeroPadding2D(padding=((1,0), (1,0)))(score_pool3c)
# Fuse things together
score_final = Add()([score4, score_pool3c])
# Final up-sampling and cropping
upsample = Conv2DTranspose(1, (4, 4), strides=4, name='upsample', use_bias=False)(score_final)
upscore = Cropping2D(((56, 56), (56, 56)))(upsample)
upscore = Cropping2D(((4, 4), (4, 4)))(upscore)
output = CrfRnnLayer(image_dims=(64, 64),
num_classes=1,
theta_alpha= config["crf_theta_alpha"], #3
theta_beta= config["crf_theta_beta"], #3
theta_gamma= config["crf_theta_gamma"], #3
num_iterations= config["crf_num_iterations"],
name='crfrnn')([upscore, img_input])
classi = Add()([upscore, output])
k = Flatten()(classi)
k = Dense(128, activation='relu')(k)
k = Dropout(.5)(k)
k = Dense(256, activation='relu')(k)
predictions = Dense(1, activation='sigmoid')(k)
# Build the model
model = Model(img_input, predictions, name='CRFVGG')
return model
from flask_httpauth import HTTPBasicAuth
from apscheduler.schedulers.background import BackgroundScheduler
from src.helper import Table, get_config
from src.db import get_current, insert_data
from src.graph_current import main as current_graph
from src.graph_nightly import main as nightly_graph
from src.graph_monthly import main as monthly_graph
import matplotlib
matplotlib.use('Agg')
# start up
app = Flask(__name__)
CONFIG = get_config()
auth = HTTPBasicAuth()
aqi_user = CONFIG['aqi_monitor']
USER_DATA = {aqi_user['authUsername']: aqi_user['authPassword']}
# initial export
print('initial export')
current_graph()
nightly_graph()
monthly_graph()
# start scheduler
scheduler = BackgroundScheduler()
scheduler.add_job(current_graph,
trigger="cron",
minute='*/5',
from keras.models import Model
from src.helper import get_config
import numpy as np
import os
import yaml
config = get_config()
def get_intermediate_output(model, layer_name, X, y=None):
intermediate_layer_model = Model(
inputs=model.input, outputs=model.get_layer(layer_name).output)
intermediate_output = intermediate_layer_model.predict(X)
return intermediate_output
def save_results(model, X, y, label, d_set, layer_name):
out_folder = config["meta"]["layers_path"]
weights_path = os.path.join(out_folder, "{}.h5".format(label))
np_path = os.path.join(out_folder, "{}_{}".format(label, d_set))
x_ = get_intermediate_output(model, layer_name, X, y)
output = np.column_stack((x_, y))
#model.save(weights_path)
np.save(np_path, output)
def save_results_wrapper(model, layer_name, label, X_train, y_train, X_test,
y_test, X_val, y_val):
def architecture(self):
config = get_config()
config = config["train"]["optimizers"]
channels, height, weight = 3, 500, 500
# Input
#input_shape = (height, weight, 3)
img_input = Input(shape=self.input_shape)
#img_input = Cropping2D((3,3))(img_input)
# Add plenty of zero padding
#x = ZeroPadding2D(padding=(218, 218))(img_input)
# block 1
x = Conv2D(strides=1, filters=32, kernel_size=3,
padding="same")(img_input)
x = BatchNormalization()(x)
x = Activation(activation="relu")(x)
x = Conv2D(strides=1, filters=32, kernel_size=3, padding="same")(x)
x = BatchNormalization()(x)
x = Activation(activation="relu")(x)
block1 = x
x = MaxPooling2D((2, 2), 2, padding="same")(x)
# block 2
x = Conv2D(strides=1, filters=16, kernel_size=3, padding="same")(x)
x = BatchNormalization()(x)
x = Activation(activation="relu")(x)
x = Conv2D(strides=1, filters=16, kernel_size=3, padding="same")(x)
x = BatchNormalization()(x)
x = Activation(activation="relu")(x)
block2 = x
x = MaxPooling2D((2, 2), 2, padding="same")(x)
block3 = x
# Deconvolution
score1 = Conv2D(1, (1, 1))(block1)
score2 = Conv2D(1, (1, 1))(block2)
score2 = Conv2DTranspose(1, (2, 2),
strides=2,
name='score2',
use_bias=False)(score2)
score3 = Conv2D(1, (1, 1))(block3)
score3 = Conv2DTranspose(1, (2, 2), strides=4)(score3)
#
score3 = BatchNormalization()(score3)
#score3 = ZeroPadding2D(2)(score3)
# Fuse things together
#score_final = Add()([score4, score_pool3c])
# Final up-sampling and cropping
score = Add()([score1, score2])
#
score = BatchNormalization()(score)
score_pool = Add()([score, score3])
#
score_pool = BatchNormalization()(score_pool)
output = CrfRnnLayer(
image_dims=(64, 64),
num_classes=1,
theta_alpha=config["crf_theta_alpha"], #3
theta_beta=config["crf_theta_beta"], #3
theta_gamma=config["crf_theta_gamma"], #3
num_iterations=config["crf_num_iterations"],
name='crfrnn')([score_pool, img_input])
#
output = BatchNormalization()(output)
classi = Add()([score_pool, output])
k = Flatten()(classi)
k = Dense(128, activation='relu')(k)
k = Dropout(.5)(k)
k = Dense(256, activation='relu')(k)
predictions = Dense(1, activation='sigmoid')(k)
# Build the model
model = Model(img_input, predictions, name='crfrnn_net')
return model
def get_exam(lesion_info,
exam='ADC',
padding=None,
base_path="data/interim/train/"):
""" Description
:type lesion_info: row of metada_labels
:param lesion_info:
:type exam: string
:param exam: the exam string to split in the description
:type padding: int
:param padding: Padding around the lesion to be retrieved
:type base_path: string
:param base_path: path to where the image files are in the project directory
:raises:
:rtype:
"""
if padding is None:
from src.helper import get_config
config = get_config()
padding = config["general"]["padding"]
exam_row = lesion_info
exam_row = exam_row.loc[exam_row.DCMSerDescr.str.contains(exam)]
if exam_row.empty:
print("No lesion found")
return None
else:
tmp_row = exam_row.iloc[0]
exam_folder = os.path.join(os.getcwd(), base_path, tmp_row.ProxID,
tmp_row.DCMSerDescr)
if (exam != 'KTrans'):
image = load_dicom_series(input_dir=exam_folder)
else:
image = load_ktrans(exam_folder)
if image is None:
return None
if (tmp_row.k < image.shape[2]):
slice_array = image[:, :, tmp_row.k]
patch = get_image_patch(slice_array, (tmp_row.i, tmp_row.j),
padding=padding)
# print(patch.shape)
else:
print("Had to cut image")
return None
if (patch.shape == (2 * padding, 2 * padding)):
return np.asarray(patch).reshape((1, 2 * padding, 2 * padding, 1))
class CurrentPlot:
""" recreate the last 3h plot """
CONFIG = get_config()
FILENAME = 'static/dyn/current.png'
def __init__(self):
self.now = datetime.now()
self.rows = self.get_data()
self.axis = None
def get_data(self):
""" export from postgres """
now_epoch = int(self.now.strftime('%s'))
last_3h = now_epoch - 3 * 60 * 60
query = ('SELECT epoch_time, aqi_value FROM aqi '
f'WHERE epoch_time > {last_3h} ORDER BY epoch_time DESC;')
db_handler = DatabaseConnect()
rows = db_handler.db_execute(query)
db_handler.db_close()
return rows
def build_title(self):
""" build title from timestamps """
time_from = datetime.fromtimestamp(self.rows[-1][0]).strftime('%H:%M')
time_until = datetime.fromtimestamp(self.rows[0][0]).strftime('%H:%M')
plt_title = f'AQI values last 3h: {time_from} - {time_until}'
return plt_title
def build_axis(self):
""" build plot axis """
rows = self.rows
x_timeline = [datetime.fromtimestamp(i[0]) for i in rows]
y_aqi_values = [int(i[1]) for i in rows]
data = {'timestamp': x_timeline, 'aqi': y_aqi_values}
df = pd.DataFrame(data)
indexed = df.set_index('timestamp')
indexed.sort_values(by=['timestamp'], inplace=True)
mean = indexed.resample('3min').mean()
mean.interpolate(method='linear',
limit=1,
inplace=True,
limit_area='inside')
mean.reset_index(level=0, inplace=True)
mean['timestamp'] = mean['timestamp'].dt.strftime('%H:%M')
mean['aqi'] = mean['aqi'].round()
plt_title = self.build_title()
# xticks
x_ticks = []
for num, i in enumerate(mean['timestamp']):
minute = int(i.split(':')[1])
if minute % 15 == 0:
x_ticks.append(num)
axis = {
'x': mean['timestamp'],
'y': mean['aqi'],
'x_ticks': x_ticks,
'plt_title': plt_title
}
self.axis = axis
def write_plt(self):
""" save plot to file """
x = self.axis['x']
y = self.axis['y']
x_ticks = self.axis['x_ticks']
# calc ticks
y_max = np.ceil(y.max() / 50) * 50 + 50
# setup plot
plt.style.use('seaborn')
plt.plot(
x,
y,
color='#313131',
)
# fill colors
plt_fill(plt, x, y)
# handle passing ticks and lables separatly
if len(x_ticks) == 2:
plt.xticks(x_ticks[0], x_ticks[1])
else:
plt.xticks(x_ticks)
plt.yticks(np.arange(0, y_max, step=50))
plt.title(self.axis['plt_title'], fontsize=20)
plt.tight_layout()
plt.savefig(self.FILENAME, dpi=300)
plt.figure()
plt.close('all')
class NightlyPlots:
""" get nightly data """
CONFIG = get_config()
def __init__(self):
self.now = datetime.now()
print('get data from db')
self.rows, self.y_rows = self.get_data()
@staticmethod
def color_colums(y):
""" helper function to color bar columns """
breakpoints = [
('#85a762', 0, 50), # good
('#d4b93c', 50, 100), # moderate
('#e96843', 100, 150), # ufsg
('#d03f3b', 150, 200), # unhealthy
('#be4173', 200, 300), # vunhealthy
('#714261', 300, 500), # hazardous
]
colors = []
for value in y:
for break_point in breakpoints:
color, min_val, max_val = break_point
if min_val < value <= max_val:
# found it
colors.append(color)
break
return colors
def get_data(self):
""" export from postgres """
# current
day_until = int(self.now.date().strftime('%s'))
day_from = day_until - 10 * 24 * 60 * 60
query = ('SELECT epoch_time, aqi_value, pm25, pm10 FROM aqi WHERE '
f'epoch_time > {day_from} AND epoch_time < {day_until} '
'ORDER BY epoch_time DESC;')
# last year
y_until = day_until - 365 * 24 * 60 * 60
y_from = y_until - 10 * 24 * 60 * 60
y_query = ('SELECT epoch_time, aqi_value FROM aqi WHERE '
f'epoch_time > {y_from} AND epoch_time < {y_until} '
'ORDER BY epoch_time DESC;')
db_handler = DatabaseConnect()
rows = db_handler.db_execute(query)
y_rows = db_handler.db_execute(y_query)
db_handler.db_close()
return rows, y_rows
def recreate_last_7(self):
""" last seven days """
day_until = int(self.now.date().strftime('%s'))
day_from = day_until - 7 * 24 * 60 * 60
rows = [i for i in self.rows if day_from < i[0] < day_until]
date_from = datetime.fromtimestamp(day_from).strftime('%d %b')
date_until = datetime.fromtimestamp(day_until).strftime('%d %b')
plt_title = f'AQI values from: {date_from} until {date_until}'
_ = LastSevenDays(rows, plt_title)
def recreate_last_3(self):
""" last three days """
_ = LastThreeDays(self.rows, self.now)
def recreate_pm_chart(self):
""" recreating pm2.5 and pm10 charts """
_ = PmGraphs(self.rows)
def recreate_hour_bar(self):
""" recreate hourly average through day bar chart """
day_until = int(self.now.date().strftime('%s'))
day_from = day_until - 3 * 24 * 60 * 60
rows = [i for i in self.rows if day_from < i[0] < day_until]
_ = HourBar(rows)
def recreate_year_comparison(self):
""" recreate year comparison chart and table for json """
_ = YearComparison(self.rows, self.y_rows)
def architecture(self):
config = get_config()
config = config["train"]["optimizers"]
# Initialize model
# Layer 1
img_input = Input(shape=self.input_shape)
x = Conv2D(96, (11, 11),
input_shape=self.input_shape,
padding='same',
activation='relu')(img_input)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPool2D(pool_size=(2, 2))(x)
block_1 = Conv2D(1, (1, 1))(x)
# Layer 2
x = Conv2D(256, (5, 5), padding='same', activation='relu')(x)
x = BatchNormalization()(x)
x = MaxPool2D(pool_size=(2, 2))(x)
# Layer 3
x = Conv2D(512, (3, 3), padding='same', activation='relu')(x)
x = BatchNormalization()(x)
x = MaxPool2D(pool_size=(2, 2))(x)
block_2 = Conv2D(1, (1, 1))(x)
# Layer 4
x = ZeroPadding2D((1, 1))(x)
x = Conv2D(512, (3, 3), padding='same', activation='relu')(x)
x = BatchNormalization()(x)
# Layer 5
x = ZeroPadding2D((1, 1))(x)
x = Conv2D(512, (3, 3), padding='same', activation='relu')(x)
x = BatchNormalization()(x)
block_3 = Conv2D(1, (1, 1))(x)
block_3 = ZeroPadding2D((2, 2))(block_3)
block_1 = Conv2DTranspose(1, (1, 1), strides=2,
name="block_1")(block_1)
block_2 = Conv2DTranspose(1, (1, 1), strides=8,
name="block_2")(block_2)
block_3 = Conv2DTranspose(1, (1, 1), strides=4,
name="block_3")(block_3)
upscore = Add()([block_1, block_3, block_2])
output = CrfRnnLayer(
image_dims=(64, 64),
num_classes=1,
theta_alpha=config["crf_theta_alpha"], #3
theta_beta=config["crf_theta_beta"], #3
theta_gamma=config["crf_theta_gamma"], #3
num_iterations=config["crf_num_iterations"],
name='crfrnn')([upscore, img_input])
classi = Add()([upscore, output])
k = Flatten()(classi)
k = Dense(128, activation='relu')(k)
k = Dropout(.5)(k)
k = Dense(256, activation='relu')(k)
predictions = Dense(1, activation='sigmoid')(k)
# Build the model
model = Model(img_input, predictions, name='CRFALEXNET')
return model
class Weather:
""" handle weather lookup from API """
CONFIG = get_config()
def __init__(self):
now = datetime.now()
self.epoch_time = int(now.strftime('%s'))
self.last_weather = self.get_weather()
def get_weather(self):
""" get weather from disk or api if too old """
try:
last_dict = self.get_cache()
except FileNotFoundError:
# create for first time
last_dict = self.get_openweather()
last_epoch = last_dict['epoch_time']
if self.epoch_time - last_epoch > 10 * 60:
print('get new weather data')
weather = self.get_openweather()
else:
print('reuse weather data')
weather = last_dict
del weather['epoch_time']
return weather
def get_openweather(self):
""" get missing weatherdata from openweathermap api """
api_key = self.CONFIG['openweathermap']['api_key']
lat = self.CONFIG['openweathermap']['lat']
lon = self.CONFIG['openweathermap']['lon']
url = ('https://api.openweathermap.org/data/2.5/weather' +
f'?&units=metric&appid={api_key}&lat={lat}&lon={lon}')
resp = requests.get(url, timeout=20).json()
weather = {
'weather_name': resp['weather'][0]['main'],
'weather_icon': resp['weather'][0]['icon'],
'wind_speed': resp['wind']['speed'],
'wind_direction': resp['wind']['deg'],
'epoch_time': self.epoch_time
}
self.write_cache(weather)
return weather
@staticmethod
def get_cache():
""" get last stored dict """
with open('static/dyn/weather.json', 'r') as f:
last = f.read()
last_dict = json.loads(last)
return last_dict
@staticmethod
def write_cache(weather):
""" update last stored value """
weather_str = json.dumps(weather)
with open('static/dyn/weather.json', 'w') as f:
f.write(weather_str)
def create_dataset(padding=None, overwrite=False):
config = get_config()
base_path = "data/interim/train/"
metadata = pd.read_csv("data/interim/train_information.csv")
s = 0
i = 0
if padding is None:
padding = config["general"]["padding"]
print(padding)
X = np.empty((1000, 2 * padding, 2 * padding, 3))
y = []
i = 0
to_iterate = metadata.drop_duplicates(["ProxID", "fid", "pos"
])[["ProxID", "fid", "ClinSig"]]
aug = iaa.Scale(0.5,
interpolation=config["preprocessing"]["interpolation"])
for tup in to_iterate.itertuples():
X_ = []
lesion_info = metadata[(metadata.ProxID == tup.ProxID)
& (metadata.fid == tup.fid)]
adc = get_exam(lesion_info, '_ADC', padding=padding)
if adc is None:
continue
t2_tse = get_exam(lesion_info, 't2_tse_tra', padding=2 * padding)
if t2_tse is None:
continue
t2_tse = np.uint8(t2_tse)
t2_tse = aug.augment_images(t2_tse)
ktrans = get_exam(lesion_info, 'KTrans', padding=padding)
if ktrans is None:
continue
X_ = np.concatenate([adc, t2_tse, ktrans], axis=3)
X[i, :, :, :] = X_
i = i + 1
y_ = 1 if tup.ClinSig else 0
y.append(y_)
X = X[:i]
print(X.shape)
if overwrite:
np.save("data/processed/X_36.npy", X)
np.save("data/processed/y_36.npy", y)
return X, y
