def extract_task1(output):
"""
Extract the data from Postgres passing a SQL query
:param output: output location
:type output: string
"""
logger.info('Extracting data to {}...'.format(output))
sql_task = """
COPY(
SELECT title, sales.country, sales.product_type, rate, discount,
start_date, end_date, EXTRACT(DAY FROM end_date-start_date) AS sale_lenght, count(*)
FROM sales
left join bookings on bookings.id = sales.id
group by title , sales.country, sales.product_type, rate, discount,
start_date , end_date, sale_lenght
order by count(*) desc
) TO
STDOUT
WITH
CSV
HEADER
DELIMITER
','
"""
extract_data(output, sql_task)
def extract_task2(output):
"""
Extract the data from Postgres passing a SQL query
:param output: output location
:type output: string
"""
logger.info('Extracting data to {}...'.format(output))
sql_task = """
COPY(
SELECT salename as booking_name, title, rate, discount, checkin, checkout,
to_char(checkin, 'Month') as month_checkin,
to_char(checkout, 'Month') as month_checkout,
to_char(checkin, 'Day') as day_checkin,
to_char(checkout, 'Day') as day_checkout,
(checkout-checkin) AS booking_length,
CASE
WHEN to_char(checkin, 'MM-DD') between '01-01' and '03-19' THEN 'Winter'
WHEN to_char(checkin, 'MM-DD') between '12-21' and '12-31' THEN 'Winter'
WHEN to_char(checkin, 'MM-DD') between '03-20' and '06-20' THEN 'Spring'
WHEN to_char(checkin, 'MM-DD') between '06-21' and '09-21' THEN 'Summer'
WHEN to_char(checkin, 'MM-DD') between '09-22' and '12-20' THEN 'Autumn'
ELSE 'UNKNOWN'
END AS season,
count(*) as bookings
FROM bookings
LEFT JOIN sales on bookings.id = sales.id
group by booking_name, title, rate, discount, checkin , checkout, month_checkin, month_checkout,
day_checkin, day_checkout, booking_length, season
order by count(*) desc
) TO
STDOUT
WITH
CSV
HEADER
DELIMITER
','
"""
extract_data(output, sql_task)
def dashboard():
per_page = 10
page = request.args.get('page', 1, type=int)
game_data = db.user_owned_games(current_user.id)
game_list = game_data[0][10 * (page - 1):(10 * (page - 1) + per_page)]
list_size = math.ceil(game_data[1] / per_page)
if list_size == 0:
list_size = 1
disp_list = functions.disp_list_generator(page, list_size)
recommended_games = db.recommendGames(current_user.id)
if page != list_size:
for i in range(per_page):
cat_list = functions.extract_data(game_list[i][4], ';')
game_list[i] += (cat_list, )
else:
for i in range(game_data[1] % per_page):
cat_list = functions.extract_data(game_list[i][4], ';')
game_list[i] += (cat_list, )
return render_template('dashboard.html',
game_list=game_list,
disp_list=disp_list,
page=page,
name=current_user.id,
recommended_games=recommended_games)
def getCitingRefData(citing_line):
rec_num = citing_line.split('|')[0]
src_doi = citing_line.split('|')[1]
src_an = citing_line.split('|')[2]
field1 = citing_line.split('|')[3]
url = citing_line.split('|')[4]
if url.strip() != "No link" or url.strip() != "None of the Citing Articles are in your subscription":
try:
record = extract_data(url)
print(record)
dst_doi = record.split('|')[11]
dst_an = record.split('|')[20]
link_rec = "|".join([rec_num, src_doi, src_an, dst_doi, dst_an])
except IOError:
record = field1 + "|||||||||||||||||||||||||"
link_rec = "|".join([rec_num, src_doi, src_an, "", ""])
else:
record = field1 + "|No link for data||||||||||||||||||||||||"
link_rec = "|".join([rec_num, src_doi, src_an, "", ""])
print("%s%s" % (field1, record), file=data_out)
print(link_rec, file=link_out)
def getCitedRefData(cited_line):
rec_num = cited_line.split('|')[0]
src_doi = cited_line.split('|')[1]
src_an = cited_line.split('|')[2]
field1 = cited_line.split('|')[3]
url = cited_line.split('|')[4]
if url.strip() != "No link" or url.strip() != "None of the Cited Articles are in your subscription":
try:
record = extract_data(url)
print(record, file=log)
dst_doi = record.split('|')[11]
dst_an = record.split('|')[20]
link_rec = "|".join([rec_num, src_doi, src_an, dst_doi, dst_an])
except IOError:
record = "|||||||||||||||||||||||||"
link_rec = "|".join([rec_num, src_doi, src_an, "", ""])
else:
record = "|No link for data||||||||||||||||||||||||"
link_rec = "|".join([rec_num, src_doi, src_an, "", ""])
print("%s%s" % (field1, record), file=data_out)
print(link_rec, file=link_out)
def game_page(id):
own_id = request.args.get('own', -1, type=int)
vote = request.args.get('vote', -1, type=int)
if vote != -1 and current_user.get_id() != None:
db.changeVote(current_user.id, id, vote)
if own_id > 0 and current_user.get_id() != None:
if db.inGamesAndNotOwned(current_user.id, own_id):
db.ownGame(current_user.id, own_id)
data = db.get_full_data(id)
date_converted = data[2].strftime("%d-%b-%Y")
data += (date_converted, )
platform_converted = functions.extract_data(data[6], ';')
data += (platform_converted, )
if current_user.get_id() != None:
flag = db.doesUserOwn(current_user.id, data[0])
if flag:
data += (True, )
vote_status = db.getVote(current_user.id, data[0])
if vote_status == 0:
data += (0, )
elif vote_status == 1:
data += (1, )
else:
data += (2, )
else:
data += (False, )
similar_games_data = db.getSimilarGames(data[10], data[0])
if current_user.get_id() != None:
return render_template('game.html',
game_data=data,
name=current_user.id,
similar_games_data=similar_games_data)
else:
return render_template('game.html',
game_data=data,
similar_games_data=similar_games_data)
st = f.make_st_spine(config.steady_state + config.ending[0])
for j, fbasal in enumerate(basal_list):
flist = []
seeds = reg_seeds
sseeds = reg_sseeds
for seed in seeds:
flist.append(fbasal + seed + add + ending)
outs_low = []
outs_high = []
l = 0
h = 0
for i, fname in enumerate(flist):
f_2.write(pars[j] + ', ' + sseeds[i] + ', ')
time_st, camkii, pkac, epac, gibg = f.extract_data(fname, 0)
dt = time_st[1] - time_st[0]
data = []
data.extend([camkii, pkac, epac, gibg])
new_data = []
for k, d in enumerate(data):
new_data.append(d / st[k] / config.max_val[0][config.keys[k]])
out = f.calculate_signature_spine(new_data)
low = sum(
(np.sign(out - threshold[0]) + 1) * dt
) * 0.5 / 1000 #sum((np.sign(out-spine_threshold[0])+1)*dt*0.5)/1000
high = sum(
(np.sign(out - threshold[1]) + 1) * dt
) * 0.5 / 1000 #sum((np.sign(out-spine_threshold[1])+1)*dt*0.5)/1000
def main(argv=None):
#Train the CNN and predict on the test and training sets."""
pr = cProfile.Profile()
pr.enable()
initialization_check()
print("----------- SETTINGS -----------")
print("Batch size: " + str(BATCH_SIZE))
print("Context size: " + str(fn.IMG_CONTEXT_SIZE))
print("Patch size: " + str(fn.IMG_PATCH_SIZE))
print("Time is termination criterion: " + str(TERMINATE_AFTER_TIME))
print("Train for: " + str(MAX_TRAINING_TIME_IN_SEC) + "s")
print("--------------------------------\n")
np.random.seed(NP_SEED)
train_data_filename = ROOT_DIR + "data/training/images/downsampled/"
train_labels_filename = ROOT_DIR + "data/training/groundtruth/downsampled/"
test_data_filename = ROOT_DIR + "data/test_set/downsampled/"
prefix = ROOT_DIR + "objects/"
patches_filename = prefix + "patches_imgs"
labels_filename = prefix + "patches_labels"
patches_balanced_filename = prefix + "patches_imgs_balanced"
labels_balanced_filename = prefix + "patches_labels_balanced"
if IMG_PATCHES_RESTORE and os.path.isfile(patches_balanced_filename +
".npy"):
if BALANCE_SIZE_OF_CLASSES:
train_data = np.load(patches_balanced_filename + ".npy")
train_labels = np.load(labels_balanced_filename + ".npy")
else:
train_data = np.load(patches_filename + ".npy")
train_labels = np.load(labels_filename + ".npy")
train_size = train_labels.shape[0]
else:
if os.path.isfile(fn.PATCHES_MEAN_PATH + ".npy"):
os.remove(fn.PATCHES_MEAN_PATH + ".npy")
print(fn.PATCHES_MEAN_PATH + ".npy" + " removed.")
train_data = fn.extract_data(train_data_filename, TRAINING_SIZE, 1)
train_labels = fn.extract_labels(train_labels_filename, TRAINING_SIZE,
1)
if IMG_PATCHES_SAVE:
np.save(patches_filename, train_data)
np.save(labels_filename, train_labels)
print("Shape of patches: " + str(train_data.shape))
print("Shape of labels: " + str(train_labels.shape))
if BALANCE_SIZE_OF_CLASSES:
def num_of_datapoints_per_class():
#Count the number of datapoints in each class
c0 = 0
c1 = 0
for i in range(len(train_labels)):
if train_labels[i][0] == 1:
c0 += 1
else:
c1 += 1
print("Number of data points per class: c0 = " + str(c0) +
" c1 = " + str(c1))
return c0, c1
# Computing per class number of data points
(c0, c1) = num_of_datapoints_per_class()
# Balancing
if IMG_PATCHES_RESTORE:
print(
"Skipping balancing - balanced data already loaded from the disk."
)
else:
print("Balancing training data.")
min_c = min(c0, c1)
idx0 = [i for i, j in enumerate(train_labels) if j[0] == 1]
idx1 = [i for i, j in enumerate(train_labels) if j[1] == 1]
new_indices = idx0[0:min_c] + idx1[0:min_c]
train_data = train_data[new_indices, :, :, :]
train_labels = train_labels[new_indices]
train_size = train_labels.shape[0]
num_of_datapoints_per_class()
if IMG_PATCHES_SAVE:
np.save(patches_balanced_filename, train_data)
np.save(labels_balanced_filename, train_labels)
PATCHES_VALIDATION = ROOT_DIR + "objects/validation_patches"
LABELS_VALIDATION = ROOT_DIR + "objects/validation_labels"
os.path.isfile(fn.PATCHES_MEAN_PATH + ".npy")
if VALIDATE:
if RESTORE_MODEL and os.path.isfile(PATCHES_VALIDATION +
".npy") and os.path.isfile(
LABELS_VALIDATION + ".npy"):
msg = "Validation data read from the disk."
validation_data = np.load(PATCHES_VALIDATION + ".npy")
validation_labels = np.load(LABELS_VALIDATION + ".npy")
else:
msg = "Validation data recreated from training data."
perm_indices = np.random.permutation(np.arange(0, len(train_data)))
validation_data = train_data[perm_indices[0:VALIDATION_SIZE]]
validation_labels = train_labels[perm_indices[0:VALIDATION_SIZE]]
np.save(PATCHES_VALIDATION, validation_data)
np.save(LABELS_VALIDATION, validation_labels)
train_data = train_data[
perm_indices[VALIDATION_SIZE:perm_indices.shape[0]]]
train_labels = train_labels[
perm_indices[VALIDATION_SIZE:perm_indices.shape[0]]]
train_size = train_labels.shape[0]
print("\n----------- VALIDATION INFO -----------")
print(msg)
print("Shape of validation set: " + str(validation_data.shape))
print("New shape of training set: " + str(train_data.shape))
print("New shape of labels set: " + str(train_labels.shape))
print("---------------------------------------\n")
# Graph variables
train_data_node = tf.placeholder(tf.float32,
shape=(BATCH_SIZE, fn.IMG_CONTEXT_SIZE,
fn.IMG_CONTEXT_SIZE, NUM_CHANNELS))
train_labels_node = tf.placeholder(tf.float32,
shape=(BATCH_SIZE, NUM_LABELS))
#weights
num_of_CNN_params_to_learn = 0
num_of_FC_params_to_learn = 0
# CONVOLUTIONAL LAYER 1
with tf.name_scope('conv1') as scope:
conv1_dim = 5
conv1_num_of_maps = 16
conv1_weights = tf.Variable(tf.truncated_normal(
[conv1_dim, conv1_dim, NUM_CHANNELS, conv1_num_of_maps],
stddev=0.1,
seed=SEED),
name='weights')
conv1_biases = tf.Variable(tf.zeros([conv1_num_of_maps]),
name='biases')
num_of_CNN_params_to_learn += conv1_dim * conv1_dim * conv1_num_of_maps
# CONVOLUTIONAL LAYER 2
with tf.name_scope('conv2') as scope:
conv2_dim = 3
conv2_num_of_maps = 32
conv2_weights = tf.Variable(tf.truncated_normal(
[conv2_dim, conv2_dim, conv1_num_of_maps, conv2_num_of_maps],
stddev=0.1,
seed=SEED),
name='weights')
conv2_biases = tf.Variable(tf.constant(0.1, shape=[conv2_num_of_maps]),
name='biases')
num_of_CNN_params_to_learn += conv2_dim * conv2_dim * conv2_num_of_maps
# CONVOLUTIONAL LAYER 3
with tf.name_scope('conv3') as scope:
conv3_dim = 3
conv3_num_of_maps = 32
conv3_weights = tf.Variable(tf.truncated_normal(
[conv3_dim, conv3_dim, conv2_num_of_maps, conv3_num_of_maps],
stddev=0.1,
seed=SEED),
name='weights')
conv3_biases = tf.Variable(tf.constant(0.1, shape=[conv3_num_of_maps]),
name='biases')
num_of_CNN_params_to_learn += conv3_dim * conv3_dim * conv3_num_of_maps
# CONVOLUTIONAL LAYER 4
with tf.name_scope('conv4') as scope:
conv4_dim = 3
conv4_num_of_maps = 64
conv4_weights = tf.Variable(tf.truncated_normal(
[conv4_dim, conv4_dim, conv3_num_of_maps, conv4_num_of_maps],
stddev=0.1,
seed=SEED),
name='weights')
conv4_biases = tf.Variable(tf.constant(0.1, shape=[conv4_num_of_maps]),
name='biases')
num_of_CNN_params_to_learn += conv4_dim * conv4_dim * conv4_num_of_maps
# FULLY CONNECTED LAYER 1
tmp_neuron_num = int(fn.IMG_CONTEXT_SIZE / 16 * fn.IMG_CONTEXT_SIZE / 16 *
conv4_num_of_maps)
with tf.name_scope('fc1') as scope:
fc1_size = 64
fc1_weights = tf.Variable(tf.truncated_normal(
[tmp_neuron_num, fc1_size], stddev=0.1, seed=SEED),
name='weights')
fc1_biases = tf.Variable(tf.constant(0.1, shape=[fc1_size]),
name='biases')
num_of_FC_params_to_learn += tmp_neuron_num * fc1_size
# FULLY CONNECTED LAYER 2
with tf.name_scope('fc1') as scope:
fc2_weights = tf.Variable(tf.truncated_normal([fc1_size, NUM_LABELS],
stddev=0.1,
seed=SEED),
name='weights')
fc2_biases = tf.Variable(tf.constant(0.1, shape=[NUM_LABELS]),
name='biases')
num_of_FC_params_to_learn += fc1_size * NUM_LABELS
if not RESTORE_MODEL:
print("----------- NUM OF PARAMS TO LEARN -----------")
print("Num of CNN params to learn: " + str(num_of_CNN_params_to_learn))
print("Num of FC params to learn: " + str(num_of_FC_params_to_learn))
print("Total num of params to learn: " +
str(num_of_CNN_params_to_learn + num_of_FC_params_to_learn))
print("----------------------------------------------\n")
def get_prediction(tf_session, img, stride):
#Get the CNN prediction for a given input image
data = fn.zero_center(
np.asarray(
fn.input_img_crop(img, fn.IMG_PATCH_SIZE, fn.IMG_BORDER_SIZE,
stride, 0)))
data_node = tf.cast(tf.constant(data), tf.float32)
prediction = tf_session.run(tf.nn.softmax(model(data_node)))
# UPSAMPLING
imgheight = img.shape[0]
imgwidth = img.shape[1]
prediction_img_per_pixel = np.zeros((imgheight, imgwidth))
count_per_pixel = np.zeros((imgheight, imgwidth))
idx = 0
for i in range(0, imgheight - fn.IMG_PATCH_SIZE + 1, stride):
for j in range(0, imgwidth - fn.IMG_PATCH_SIZE + 1, stride):
prediction_img_per_pixel[
j:j + fn.IMG_PATCH_SIZE,
i:i + fn.IMG_PATCH_SIZE] += prediction[idx][1]
count_per_pixel[j:j + fn.IMG_PATCH_SIZE,
i:i + fn.IMG_PATCH_SIZE] += 1.0
idx += 1
prediction = np.zeros((imgheight * imgwidth, 2))
idx = 0
for i in range(imgheight):
for j in range(imgwidth):
prediction[idx][
1] = prediction_img_per_pixel[j][i] / count_per_pixel[j][i]
prediction[idx][0] = 1.0 - prediction[idx][1]
idx += 1
return prediction
def get_prediction_with_overlay(tf_session,
input_path,
output_path_overlay,
output_path_raw,
truth_input_path=None):
#Get the CNN prediction overlaid on the original image for a given input file
def label_to_binary_img(imgwidth, imgheight, w, h, labels):
array_labels = np.zeros([imgwidth, imgheight])
idx = 0
for i in range(0, imgheight, h):
for j in range(0, imgwidth, w):
if labels[idx][0] > 0.5:
l = 0
else:
l = 1
array_labels[j:j + w, i:i + h] = l
idx += 1
return array_labels
def label_to_img(imgwidth, imgheight, w, h, labels):
array_labels = np.zeros([imgwidth, imgheight])
idx = 0
for i in range(0, imgheight, h):
for j in range(0, imgwidth, w):
array_labels[j:j + w, i:i + h] = labels[idx][1]
idx += 1
return array_labels
def make_img_overlay(img, predicted_img, true_img=None):
def img_float_to_uint8(img):
rimg = img - np.min(img)
rimg = (rimg / np.max(rimg) * PIXEL_DEPTH).round().astype(
np.uint8)
return rimg
w = img.shape[0]
h = img.shape[1]
color_mask = np.zeros((w, h, 3), dtype=np.uint8)
color_mask[:, :, 0] = predicted_img * PIXEL_DEPTH
if true_img is not None:
color_mask[:, :, 1] = true_img * PIXEL_DEPTH
img8 = img_float_to_uint8(img)
background = Image.fromarray(img8, 'RGB').convert("RGBA")
overlay = Image.fromarray(color_mask, 'RGB').convert("RGBA")
new_img = Image.blend(background, overlay, 0.2)
return new_img
def pixels_to_patches(img,
round=False,
foreground_threshold=0.5,
stride=fn.IMG_PATCH_SIZE):
res_img = np.zeros(img.shape)
for i in range(0, img.shape[0], stride):
for j in range(0, img.shape[1], stride):
tmp = np.zeros((stride, stride))
tmp[0:stride, 0:stride] = img[j:j + stride, i:i + stride]
tmp[tmp < 0.5] = 0
tmp[tmp >= 0.5] = 1
res_img[j:j + stride, i:i + stride] = np.mean(tmp)
if round:
if res_img[j, i] >= foreground_threshold:
res_img[j:j + stride, i:i + stride] = 1
else:
res_img[j:j + stride, i:i + stride] = 0
return res_img
# Read images from disk
img = mpimg.imread(input_path)
img_truth = None
if truth_input_path is not None:
img_truth = mpimg.imread(truth_input_path)
# Get prediction
stride = fn.IMG_PATCH_SIZE
prediction = get_prediction(tf_session, img, stride)
# Show per pixel probabilities
prediction_as_per_pixel_img = label_to_img(img.shape[0], img.shape[1],
1, 1, prediction)
# Show per patch probabilities
prediction_as_img = pixels_to_patches(prediction_as_per_pixel_img)
# Rounded to 0 / 1 - per pixel
prediction_as_binary_per_pixel_img = label_to_binary_img(
img.shape[0], img.shape[1], 1, 1, prediction)
# Round to 0 / 1 - per patch
prediction_as_binary_img = pixels_to_patches(
prediction_as_per_pixel_img, True)
# Save output to disk
# Overlay
oimg = make_img_overlay(img, prediction_as_binary_per_pixel_img,
img_truth)
oimg.save(output_path_overlay + "_pixels.png")
oimg2 = make_img_overlay(img, prediction_as_binary_img, img_truth)
oimg2.save(output_path_overlay + "_patches.png")
# Raw image
scipy.misc.imsave(
output_path_raw.replace("/raw/", "/high_res_raw/") + "_pixels.png",
prediction_as_per_pixel_img)
scipy.misc.imsave(output_path_raw + "_patches.png", prediction_as_img)
return prediction, prediction_as_binary_img
def validate(validation_model, labels):
print("\n --- Validation ---")
prediction = s.run(tf.nn.softmax(validation_model))
err = error_rate(prediction, labels)
acc = 100 - err
print("Validation set size: %d" % VALIDATION_SIZE)
print("VALIDATION ACCURACY: %.1f%%" % acc)
print("Error: %.1f%%" % err)
now = time.time() - st_time
log_file.write(str(now) + "," + str(acc) + "\n")
log_file.flush()
print("--------------------")
return err
def model(data, train=False):
#Define the CNN
# CONV. LAYER 1
conv1 = tf.nn.conv2d(data,
conv1_weights,
strides=[1, 1, 1, 1],
padding='SAME')
relu1 = tf.nn.relu(tf.nn.bias_add(conv1, conv1_biases))
norm1 = tf.nn.lrn(relu1)
pool1 = tf.nn.max_pool(norm1,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
# CONV. LAYER 2
conv2 = tf.nn.conv2d(pool1,
conv2_weights,
strides=[1, 1, 1, 1],
padding='SAME')
relu2 = tf.nn.relu(tf.nn.bias_add(conv2, conv2_biases))
norm2 = tf.nn.lrn(relu2)
pool2 = tf.nn.max_pool(norm2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
# CONV. LAYER 3
conv3 = tf.nn.conv2d(pool2,
conv3_weights,
strides=[1, 1, 1, 1],
padding='SAME')
relu3 = tf.nn.relu(tf.nn.bias_add(conv3, conv3_biases))
norm3 = tf.nn.lrn(relu3)
pool3 = tf.nn.max_pool(norm3,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
# CONV. LAYER 4
conv4 = tf.nn.conv2d(pool3,
conv4_weights,
strides=[1, 1, 1, 1],
padding='SAME')
relu4 = tf.nn.relu(tf.nn.bias_add(conv4, conv4_biases))
norm4 = tf.nn.lrn(relu4)
pool4 = tf.nn.max_pool(norm4,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
conv_out = pool4
# Reshape to 2D
pool_shape = conv_out.get_shape().as_list()
reshape = tf.reshape(
conv_out,
[pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3]])
# FC 1
hidden = tf.nn.relu(tf.matmul(reshape, fc1_weights) + fc1_biases)
# FINAL ACTIVATION
out = tf.sigmoid(tf.matmul(hidden, fc2_weights) + fc2_biases)
def get_image_summary(img, idx=0):
"""Make an image summary for 4d tensor image with index idx."""
V = tf.slice(img, (0, 0, 0, idx), (1, -1, -1, 1))
img_w = img.get_shape().as_list()[1]
img_h = img.get_shape().as_list()[2]
min_value = tf.reduce_min(V)
V = V - min_value
max_value = tf.reduce_max(V)
V = V / (max_value * PIXEL_DEPTH)
V = tf.reshape(V, (img_w, img_h, 1))
V = tf.transpose(V, (2, 0, 1))
V = tf.reshape(V, (-1, img_w, img_h, 1))
return V
if train:
tf.summary.image('summary_data', get_image_summary(data))
tf.summary.image('summary_conv1', get_image_summary(conv1))
tf.summary.image('summary_pool1', get_image_summary(pool1))
tf.summary.image('summary_conv2', get_image_summary(conv2))
tf.summary.image('summary_pool2', get_image_summary(pool2))
tf.summary.image('summary_conv3', get_image_summary(conv3))
tf.summary.image('summary_pool3', get_image_summary(pool3))
tf.summary.image('summary_conv4', get_image_summary(conv4))
tf.summary.image('summary_pool4', get_image_summary(pool4))
tf.summary.histogram('weights_conv1', conv1_weights)
tf.summary.histogram('weights_conv2', conv2_weights)
tf.summary.histogram('weights_conv3', conv3_weights)
tf.summary.histogram('weights_conv4', conv4_weights)
tf.summary.histogram('weights_FC1', fc1_weights)
tf.summary.histogram('weights_FC2', fc2_weights)
return out
#losses
logits = model(train_data_node, True)
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=train_labels_node))
tf.summary.scalar('loss', loss)
cumulative_loss = tf.Variable(1.0)
loss_window = np.zeros(LOSS_WINDOW_SIZE)
index_loss_window = 0
tf.summary.scalar('loss_smoothed', cumulative_loss)
# regularizers = (tf.nn.l2_loss(fc1_weights) + tf.nn.l2_loss(fc1_biases) +
# tf.nn.l2_loss(fc2_weights) + tf.nn.l2_loss(fc2_biases))
# loss = tf.add(loss, 5e-4 * regularizers)
#IN SAMPLE ERROR
error_insample_tensor = tf.Variable(0)
tf.summary.scalar('error_insample_smoothed', error_insample_tensor)
insample_error_window = np.zeros(LOSS_WINDOW_SIZE)
index_insample_error_window = 0
# VALIDATION ERROR
error_validation_tensor = tf.Variable(0)
tf.summary.scalar('error_validation', error_validation_tensor)
# validation model
if VALIDATE:
data_node = tf.cast(tf.constant(np.asarray(validation_data)),
tf.float32)
validation_model = model(data_node)
#SUMMARY OF WEIGHTS
all_params_node = [
conv1_weights, conv1_biases, conv2_weights, conv2_biases,
conv3_weights, conv3_biases, fc1_weights, fc1_biases, fc2_weights,
fc2_biases
]
all_params_names = [
'conv1_weights', 'conv1_biases', 'conv2_weights', 'conv2_biases',
'conv3_weights', 'conv3_biases', 'fc1_weights', 'fc1_biases',
'fc2_weights', 'fc2_biases'
]
all_grads_node = tf.gradients(loss, all_params_node)
all_grad_norms_node = []
for i in range(0, len(all_grads_node)):
norm_grad_i = tf.global_norm([all_grads_node[i]])
all_grad_norms_node.append(norm_grad_i)
tf.summary.scalar(all_params_names[i], norm_grad_i)
#optimiser
batch = tf.Variable(0)
learning_rate = tf.Variable(BASE_LEARNING_RATE)
tf.summary.scalar('learning_rate', learning_rate)
optimizer = tf.train.AdamOptimizer(learning_rate,
epsilon=0.1).minimize(loss,
global_step=batch)
# Predictions for the minibatch, validation set and test set
train_prediction = tf.nn.softmax(logits)
# Add ops to save and restore all the variables.
saver = tf.train.Saver()
# run the session
with tf.Session() as s:
if RESTORE_MODEL:
print("### MODEL RESTORED ###")
else:
tf.initialize_all_variables().run()
print("### MODEL INITIALIZED ###")
print("### TRAINING STARTED ###")
training_indices = range(train_size)
start = time.time()
run_training = True
iepoch = 0
batch_index = 1
while run_training:
perm_indices = np.random.permutation(training_indices)
for step in range(int(train_size / BATCH_SIZE)):
if not run_training:
break
offset = (batch_index * BATCH_SIZE) % (train_size -
BATCH_SIZE)
batch_indices = perm_indices[offset:(offset + BATCH_SIZE)]
# Compute the offset of the current minibatch in the data.
batch_data = train_data[batch_indices, :, :, :]
batch_labels = train_labels[batch_indices]
# The dictionary maps batch data (as np array) to node in the graph
feed_dict = {
train_data_node: batch_data,
train_labels_node: batch_labels
}
# Run the operations
_, l, lr, predictions = s.run(
[optimizer, loss, learning_rate, train_prediction],
feed_dict=feed_dict)
# Update cumulative loss
loss_window[index_loss_window] = l
index_loss_window = (index_loss_window +
1) % loss_window.shape[0]
# Update insample error
insample_error_window[
index_insample_error_window] = error_rate(
predictions, batch_labels)
index_insample_error_window = (
index_insample_error_window +
1) % insample_error_window.shape[0]
if batch_index % RECORDING_STEP == 0 and batch_index > 0:
closs = np.mean(loss_window)
s.run(cumulative_loss.assign(closs))
insample_error = error_rate(predictions, batch_labels)
s.run(error_insample_tensor.assign(insample_error))
if VALIDATE and batch_index % VALIDATION_STEP == 0:
validation_err = validate(validation_model,
validation_labels)
s.run(
error_validation_tensor.assign(validation_err))
print("\nEpoch: %d, Batch #: %d" % (iepoch, step))
print("Global step: %d" %
(batch_index * BATCH_SIZE))
print("Time elapsed: %.3fs" % (time.time() - start))
print("Minibatch loss: %.6f" % l)
print("Cumulative loss: %.6f" % closs)
print("Learning rate: %.6f" % lr)
print("Minibatch insample error: %.1f%%" %
insample_error)
sys.stdout.flush()
batch_index += 1
if TERMINATE_AFTER_TIME and time.time(
) - start > MAX_TRAINING_TIME_IN_SEC:
run_training = False
iepoch += 1
if not TERMINATE_AFTER_TIME:
run_training = False
prefix_results = ROOT_DIR + "results/CNN_Output/"
if VISUALIZE_PREDICTION_ON_TRAINING_SET:
print("--- Visualizing prediction on training set ---")
prediction_training_dir = prefix_results + "training/"
if not os.path.isdir(prediction_training_dir):
os.mkdir(prediction_training_dir)
limit = TRAINING_SIZE + 1 if VISUALIZE_NUM == -1 else VISUALIZE_NUM
for i in range(1, limit):
print("Image: " + str(i))
img_name = "satImage_%.3d" % i
input_path = train_data_filename + img_name + ".png"
truth_path = train_labels_filename + img_name + ".png"
output_path_overlay = prediction_training_dir + "overlay_" + img_name
output_path_raw = prediction_training_dir + "raw/" + "raw_" + img_name
get_prediction_with_overlay(s, input_path, output_path_overlay,
output_path_raw, truth_path)
if VALIDATE:
validation_err = validate(validation_model, validation_labels)
if RUN_ON_TEST_SET:
print("--- Running prediction on test set ---")
prediction_test_dir = prefix_results + "test/"
if not os.path.isdir(prediction_test_dir):
os.mkdir(prediction_test_dir)
for i in range(1, TEST_SIZE + 1):
print("Test img: " + str(i))
# Visualization
img_name = "test_" + str(i)
input_path = test_data_filename + img_name + ".png"
output_path_overlay = prediction_test_dir + "overlay_" + img_name
output_path_raw = prediction_test_dir + "raw/" + "raw_" + img_name
(_, prediction_as_img) = get_prediction_with_overlay(
s, input_path, output_path_overlay, output_path_raw)
prediction_as_img = prediction_as_img.astype(np.int)
# End profiling and save stats
pr.disable()
s = io.StringIO()
sortby = 'cumulative'
stream = open('profile.txt', 'w')
ps = pstats.Stats(pr, stream=stream).sort_stats(sortby)
ps.print_stats()
out2_file = "/home/usha/python/log/cited_ref_link.txt"
out = open(out_file, "a")
out2 = open(out2_file, "a")
print("Extracting cited reference data....")
for cnt, line in enumerate(file(in_file, "r")):
rec_num = line.split('|')[0]
src_doi = line.split('|')[1]
src_an = line.split('|')[2]
field1 = line.split('|')[3]
url = line.split('|')[4]
if url.strip() != "No link" and url.strip(
) != "None of the Citing Articles are in your subscription":
try:
print("Extract record")
record = extract_data(url)
print(record)
dst_doi = record.split('|')[11]
dst_an = record.split('|')[20]
link_rec = "|".join([rec_num, src_doi, src_an, dst_doi, dst_an])
except IOError:
print("Test")
record = field1 + "|||||||||||||||||||||||||"
link_rec = "|".join([rec_num, src_doi, src_an, "", ""])
else:
record = field1 + "|No link for data||||||||||||||||||||||||"
link_rec = "|".join([rec_num, src_doi, src_an, "", ""])
print("%s%s" % (field1, record), file=out)
print(link_rec, file=out2)
out.close()
def index():
per_page = 10
page = request.args.get('page', 1, type=int)
query = request.args.get('query', '', type=str)
query_type = request.args.get('query_type', 0, type=int)
own_id = request.args.get('own', -1, type=int)
if own_id > 0 and current_user.get_id() != None:
if db.inGamesAndNotOwned(current_user.id, own_id):
db.ownGame(current_user.id, own_id)
if query == '':
game_data = db.get_game_data('', 0)
else:
game_data = db.get_game_data(query, query_type)
game_list = game_data[0][10 * (page - 1):(10 * (page - 1) + per_page)]
list_size = math.ceil(game_data[1] / per_page)
if list_size == 0:
list_size = 1
disp_list = functions.disp_list_generator(page, list_size)
if page != list_size:
for i in range(per_page):
cat_list = functions.extract_data(game_list[i][4], ';')
game_list[i] += (cat_list, )
if current_user.get_id() != None:
flag = db.doesUserOwn(current_user.id, game_list[i][0])
if flag:
game_list[i] += (True, )
else:
game_list[i] += (False, )
else:
for i in range(game_data[1] % per_page):
cat_list = functions.extract_data(game_list[i][4], ';')
game_list[i] += (cat_list, )
if current_user.get_id() != None:
flag = db.doesUserOwn(current_user.id, game_list[i][0])
if flag:
game_list[i] += (True, )
else:
game_list[i] += (False, )
if query == '':
if current_user.get_id() != None:
return render_template('index.html',
game_list=game_list,
disp_list=disp_list,
page=page,
name=current_user.id)
else:
return render_template('index.html',
game_list=game_list,
disp_list=disp_list,
page=page)
else:
if query_type == 1:
if current_user.get_id() != None:
return render_template('index.html',
game_list=game_list,
disp_list=disp_list,
page=page,
filter=query,
query_type=1,
name=current_user.id)
else:
return render_template('index.html',
game_list=game_list,
disp_list=disp_list,
page=page,
filter=query,
query_type=1)
elif query_type == 2:
if current_user.get_id() != None:
return render_template('index.html',
game_list=game_list,
disp_list=disp_list,
page=page,
filter=query,
query_type=2,
name=current_user.id)
else:
return render_template('index.html',
game_list=game_list,
disp_list=disp_list,
page=page,
filter=query,
query_type=2)
elif query_type == 3:
if current_user.get_id() != None:
return render_template('index.html',
game_list=game_list,
disp_list=disp_list,
page=page,
filter=query,
query_type=3,
name=current_user.id)
else:
return render_template('index.html',
game_list=game_list,
disp_list=disp_list,
page=page,
filter=query,
query_type=3)