def menu():
"""Displays the menu and redirects to underlaying options"""
clear()
### Get desired option
option = input(MAIN_MENU)
try:
option = OPTIONS[int(option)]
except ValueError:
pass
except KeyError:
option = 'UNDEFINED'
if option.upper() == 'C':
### Create
cm = CreateModule()
cm.setup()
sleep(1)
menu()
elif option.upper() == 'S':
### Search
sm = SearchModule()
entries = sm.setup()
vm = ViewModule(entries)
vm.load_entries()
menu()
elif option.upper() == 'Q':
### Quit
finalize()
else:
### Undefined option
print('Undefined option. Restarting!')
sleep(1)
menu()
def plot_box(xs, ys,
xaxis_label=None, yaxis_label=None,
x_sci=False, y_sci=True,
name=None):
fig, ax = plt.subplots()
ax = sns.boxplot(x=xs, y=ys, linewidth=1, fliersize=1)
utils.set_sci_axis(ax, x_sci, y_sci)
utils.set_axis_labels(ax, xaxis_label, yaxis_label)
utils.finalize(ax)
utils.save_pdf(name)
def plot_box(xs,
ys,
data,
xaxis_label=None,
yaxis_label=None,
x_sci=False,
y_sci=True,
name=None):
fig, ax = plt.subplots()
ax = sns.boxplot(x=xs, y=ys, data=data, linewidth=1, fliersize=6)
utils.set_sci_axis(ax, x_sci, y_sci)
utils.set_axis_labels(ax, xaxis_label, yaxis_label)
utils.finalize(ax)
utils.save_fig(name)
def plot_heatmap(x, y, z,
xaxis_label=None, yaxis_label=None, mapaxis_label=None,
x_sci=True, y_sci=True, name=None):
assert utils.is_list(x) == utils.is_list(y) == utils.is_list(z)
fig, ax = plt.subplots()
ax.hexbin(x, y, C=z, gridsize=30, cmap=plt.get_cmap('Blues'), bins=None)
utils.set_sci_axis(ax, x_sci, y_sci)
pcm = ax.get_children()[2]
cb = plt.colorbar(pcm, ax=ax)
cb.set_label(mapaxis_label)
utils.set_axis_labels(ax, xaxis_label, yaxis_label)
utils.finalize(ax)
utils.save_pdf(name)
def preprocessor(filename, genre):
with open(filename, "r", encoding='utf-8') as f:
data = f.read().split("\n")
data.remove("")
chunks, clusters, ner, srls, zero_ants = get_tags(data)
doc_data = utils.finalize(chunks, clusters, ner, srls, zero_ants,
genre)
return doc_data
def plot_bar(x, y, hue=None, hue_count=1,
line_values=None, line_label=None, legend=True,
xaxis_label=None, yaxis_label=None,
xticks=None, y_lim=None,
x_sci=True, y_sci=True,
fig_size=None,
y_err=None,
name=None):
fig, ax = plt.subplots()
if fig_size and isinstance(fig_size, list) and len(fig_size) > 0:
if len(fig_size) == 1:
fig.set_figwidth(fig_size[0])
else:
fig.set_figwidth(fig_size[0])
fig.set_figheight(fig_size[1])
bar_width = 0.2
new_x = [x_ + bar_width for x_ in x]
ticks_x = [x_ + 0.5*bar_width for x_ in new_x]
if y_err:
ax.errorbar(ticks_x, y, yerr=y_err, fmt='o', ecolor='r', capthick=1, elinewidth=1)
plt.bar(new_x, y, width=bar_width)
if line_values and utils.is_list(line_values):
x_set = set(x)
if len(line_values) == len(x_set):
if line_label:
ax.plot(ax.get_xticks(), line_values, label=line_label)
hue_count += 1
else:
ax.plot(ax.get_xticks(), line_values)
if xticks and isinstance(xticks, list):
plt.xticks(ticks_x, xticks)
if y_lim and isinstance(y_lim, list) and len(y_lim) > 0:
if len(y_lim) == 1:
plt.ylim(ymin=y_lim[0])
else:
plt.ylim(ymin=y_lim[0])
plt.ylim(ymax=y_lim[1])
if legend:
utils.set_legend(ncol=hue_count)
utils.set_sci_axis(ax, x_sci, y_sci)
utils.set_axis_labels(ax, xaxis_label, yaxis_label)
utils.finalize(ax, x_grid=False)
utils.save_fig(name)
def plot_two_bars(ys, labels,
legend=True,
xaxis_label=None,
yaxis_label=None,
xticks=None, y_lim=None,
x_sci=True, y_sci=True,
fig_size=None,
name=None):
fig, ax = plt.subplots()
if fig_size and isinstance(fig_size, list) and len(fig_size) > 0:
if len(fig_size) == 1:
fig.set_figwidth(fig_size[0])
else:
fig.set_figwidth(fig_size[0])
fig.set_figheight(fig_size[1])
bar_width = 0.2
x = range(len(ys))
new_x1 = [x_ for x_ in x]
new_x2 = [x_ + bar_width for x_ in x]
rects1 = ax.bar(new_x1, ys[0], width=bar_width, color='b')
rects2 = ax.bar(new_x2, ys[1], width=bar_width)
if xticks and isinstance(xticks, list):
plt.xticks([x_ + bar_width for x_ in new_x1], xticks)
if y_lim and isinstance(y_lim, list) and len(y_lim) > 0:
if len(y_lim) == 1:
plt.ylim(ymin=y_lim[0])
else:
plt.ylim(ymin=y_lim[0])
plt.ylim(ymax=y_lim[1])
if legend:
ax.legend((rects1[0], rects2[0]), (labels[0], labels[1]),
fontsize=28, frameon=False,
bbox_to_anchor=(0, 0.95, 1.2, .10), handlelength=0.5, handletextpad=0.2,
loc=3, ncol=2, mode="expand",
borderaxespad=0.)
utils.set_sci_axis(ax, x_sci, y_sci)
utils.set_axis_labels(ax, xaxis_label, yaxis_label)
utils.finalize(ax, x_grid=False)
utils.save_fig(name)
def plot_line(xs,
ys,
line_labels=None,
xaxis_label=None,
yaxis_label=None,
xticks=None,
vlines=None,
vlines_kwargs=None,
hlines=None,
hlines_kwargs=None,
x_sci=True,
y_sci=True,
y_lim=None,
x_lim=None,
legend_top=True,
ls_cycle=False,
marker_size=0,
x_grid=True, y_grid=True,
fig_size=None,
name=None, draw_arrow=False):
multiple_x = utils.is_list_of_list(xs)
multiple_y = utils.is_list_of_list(ys)
multiple_line_label = utils.is_list(line_labels)
assert multiple_x == multiple_y == multiple_line_label
fig, ax = plt.subplots()
if fig_size and isinstance(fig_size, list) and len(fig_size) > 0:
if len(fig_size) == 1:
fig.set_figwidth(fig_size[0])
else:
fig.set_figwidth(fig_size[0])
fig.set_figheight(fig_size[1])
ls_cycler = utils.get_line_styles_cycler(ls_cycle)
ms_cycler = utils.get_marker_styles_cycler(marker_size > 0)
if multiple_x:
for x, y, line_label in zip(xs, ys, line_labels):
ax.plot(x, y, label=line_label, ls=next(ls_cycler), marker=next(ms_cycler), markersize=marker_size)
else:
ax.plot(xs, ys, label=line_labels)
if xticks and isinstance(xticks, list):
x = xs[0] if multiple_x else xs
plt.xticks(x, xticks)
if y_lim and isinstance(y_lim, list) and len(y_lim) > 0:
if len(y_lim) == 1:
plt.ylim(ymin=y_lim[0])
else:
plt.ylim(ymin=y_lim[0])
plt.ylim(ymax=y_lim[1])
if x_lim and isinstance(x_lim, list) and len(x_lim) > 0:
if len(x_lim) == 1:
plt.xlim(xmin=x_lim[0])
else:
plt.xlim(xmin=x_lim[0])
plt.xlim(xmax=x_lim[1])
ncol = len(xs) if multiple_x else 1
utils.set_legend(legend_top, ncol)
utils.set_sci_axis(ax, x_sci, y_sci)
utils.set_axis_labels(ax, xaxis_label, yaxis_label)
vlines = vlines or []
for xvline in vlines:
with ALTERNATIVE_PALETTE:
plt.axvline(x=xvline, **vlines_kwargs)
hlines = hlines or []
for yhline in hlines:
with ALTERNATIVE_PALETTE:
plt.axhline(y=yhline, **hlines_kwargs)
utils.finalize(ax, x_grid=x_grid, y_grid=y_grid)
utils.save_fig(name)
ones,
imed_score > prob_normality,
label="Detected Anomaly",
alpha=0.5,
color="C0")
ax[1].set_ylim(1e-4, 1.1)
ax[1].set_ylabel("Normality")
bbox = {"boxstyle": "round", "pad": 0.3, "fc": "white", "ec": "gray", "lw": 1}
for a, l in zip(ax, 'ABCD'):
a.annotate(l, xy=(0.05, 0.8), xycoords='axes fraction', bbox=bbox)
ax[1].set_yscale("log")
ax[1].legend(loc="lower left")
ax[2].set_yscale("log")
ax[2].legend(loc="lower left")
#ax[3].set_yscale("log")
#ax[3].legend(loc="lower left")
ax[0].annotate("Year 1", xy=(0.08, 1.02), xycoords="axes fraction")
ax[0].annotate("Year 2", xy=(0.27, 1.02), xycoords="axes fraction")
ax[0].annotate("Year 3", xy=(0.46, 1.02), xycoords="axes fraction")
ax[0].annotate("Year 4", xy=(0.67, 1.02), xycoords="axes fraction")
ax[0].annotate("Year 5", xy=(0.90, 1.02), xycoords="axes fraction")
for a in ax:
a.set_xticks(np.arange(0, indices[-1], 365))
a.set_xlim(plot_start, plot_end)
a.grid(True)
finalize(args, fig, ax, loc="lower left")
if option.upper() == 'C':
### Create
cm = CreateModule()
cm.setup()
sleep(1)
menu()
elif option.upper() == 'S':
### Search
sm = SearchModule()
entries = sm.setup()
vm = ViewModule(entries)
vm.load_entries()
menu()
elif option.upper() == 'Q':
### Quit
finalize()
else:
### Undefined option
print('Undefined option. Restarting!')
sleep(1)
menu()
if __name__ == '__main__':
init_db()
### Clean exits
try:
menu()
except KeyboardInterrupt:
finalize()
def plot_scatter(xs,
ys,
line_labels=None,
xaxis_label=None,
yaxis_label=None,
xticks=None,
vlines=None,
vlines_kwargs=None,
hlines=None,
hlines_kwargs=None,
x_sci=True,
y_sci=True,
y_lim=None,
legend_top=True,
fig_size=None,
ls_cycle=False,
name=None):
multiple_x = utils.is_list_of_list(xs)
multiple_y = utils.is_list_of_list(ys)
multiple_line_label = utils.is_list(line_labels)
assert multiple_x == multiple_y == multiple_line_label
fig, ax = plt.subplots()
if fig_size and isinstance(fig_size, list) and len(fig_size) > 0:
if len(fig_size) == 1:
fig.set_figwidth(fig_size[0])
else:
fig.set_figwidth(fig_size[0])
fig.set_figheight(fig_size[1])
colors = iter(cm.rainbow(np.linspace(0, 1, len(ys))))
if multiple_x:
for x, y, line_label in zip(xs, ys, line_labels):
ax.scatter(x, y, label=line_label, c=next(colors))
else:
ax.scatter(xs, ys, label=line_labels, c=next(colors))
if xticks and isinstance(xticks, list):
x = xs[0] if multiple_x else xs
plt.xticks(x, xticks)
if y_lim and isinstance(y_lim, list) and len(y_lim) > 0:
if len(y_lim) == 1:
plt.ylim(ymin=y_lim[0])
else:
plt.ylim(ymin=y_lim[0])
plt.ylim(ymax=y_lim[1])
plt.xlim(xmin=0)
ncol = len(xs) if multiple_x else 1
utils.set_legend(legend_top, ncol)
utils.set_sci_axis(ax, x_sci, y_sci)
utils.set_axis_labels(ax, xaxis_label, yaxis_label)
vlines = vlines or []
for xvline in vlines:
with ALTERNATIVE_PALETTE:
plt.axvline(x=xvline, **vlines_kwargs)
hlines = hlines or []
for yhline in hlines:
with ALTERNATIVE_PALETTE:
plt.axhline(y=yhline, **hlines_kwargs)
utils.finalize(ax)
utils.save_fig(name)
def run_next_job(self):
# Updates the earliest job from status 0 to status 1, and returns the gid of the updated row. Returns nothing
# if no pending jobs are available.
sql = """update job set status = 'in progress'
where gid in (select min(gid) from job where status = 'ready')
returning gid"""
current_job = db.exec_sql(sql, [], True)
self.update_status("processing tiles")
if type(current_job) is tuple:
self.job_id = current_job[0]
self.thread_list = []
self.log_message("started job")
self.work_path = conf.work_path + str(self.job_id) + "/"
if not os.path.exists(self.work_path):
os.makedirs(self.work_path)
tile_list = self.get_tiles_to_process()
minx, miny, maxx, maxy = self.get_bounds()
if conf.num_threads > 1:
if len(tile_list) < conf.num_threads:
self.num_threads = len(tile_list)
else:
self.num_threads = conf.num_threads
tile_count_step = len(tile_list) // self.num_threads # Integer division to get floow
args = [[] for x in range(conf.num_threads)]
for i in range(len(tile_list)):
args[i % self.num_threads].append(tile_list[i])
pool = Pool(self.num_threads)
pool.map(process_tiles, args)
pool.close()
pool.join()
else:
process_tiles(tile_list)
self.update_status("saving clip file")
sql = "select geom_txt from job where gid = {0}".format(self.job_id)
wkt = db.exec_sql(sql, [], True)[0]
clip_poly = self.work_path + "clip_poly.shp"
utils.save_clip_poly(wkt, clip_poly)
# self.update_status("mosaicking tiles")
# utils.mosaic_tiles(self.work_path + "*_dsm.tif", self.work_path + "mosaic_dsm.tif", minx, miny, maxx, maxy,
# clip_poly, self.job_id)
# utils.mosaic_tiles(self.work_path + "*_height.tif", self.work_path + "mosaic_height.tif", minx, miny, maxx,
# maxy, clip_poly, self.job_id)
# # utils.mosaic_tiles(self.work_path + "*_intensity.tif", self.work_path + "mosaic_intensity.tif", minx, miny, maxx, maxy, clip_poly)
# utils.mosaic_tiles(self.work_path + "*_dem.tif", self.work_path + "mosaic_dem.tif", minx, miny, maxx, maxy,
# clip_poly, self.job_id)
self.update_status("finalizing output")
tree_file = self.work_path + "mosaic_trees.shp"
utils.merge_tiles(self.work_path + "*_trees.shp", tree_file, clip_poly)
utils.finalize(tree_file, clip_poly, self.job_id)
# bldgs_file = self.work_path + "mosaic_bldgs.shp"
# utils.merge_tiles(self.work_path + "*_bldgs.shp", bldgs_file, clip_poly, 10)
# utils.finalize(bldgs_file, clip_poly, self.job_id)
# impervious_file = self.work_path + "mosaic_impervious.shp"
# utils.merge_tiles(self.work_path + "*_impervious.shp", impervious_file, clip_poly, 10)
# utils.finalize(impervious_file, clip_poly, self.job_id)
# contours_file = self.work_path + "mosaic_contours.shp"
# utils.contours(self.work_path + "mosaic_dem.tif", contours_file, clip_poly)
# utils.finalize(contours_file, clip_poly, self.job_id)
for f_name in gl.glob(self.work_path + "*.txt"):
out_file = conf.output_path + str(job_id) + "_" + os.path.basename(f_name)
cmd = "copy {0} {1}".format(f_name.replace("/", "\\"), out_file.replace("/", "\\"))
utils.exec_command_line(cmd, shell_flag=True)
self.update_status("job complete!")
# utils.mosaic_tiles(self.work_path + "*_trees.tif", self.work_path + "mosaic_trees.tif", minx, miny, maxx, maxy)
# self.update_status("generating map tiles")
# utils.create_output_tiles(self.work_path + "mosaic_dsm.tif", self.work_path + "dsm/")
# utils.create_output_tiles(self.work_path + "mosaic_height.tif", self.work_path + "height/")
# # utils.create_output_tiles(self.work_path + "mosaic_intensity.tif", self.work_path + "intensity/")
# utils.create_output_tiles(self.work_path + "mosaic_dem.tif", self.work_path + "dem/")
# # utils.create_output_tiles(self.work_path + "mosaic_trees.tif", self.work_path + "trees/")
else:
self.job_id = None
self.log_message("No jobs found")
lines = []
for t in tile_list:
line = "\t".join([str(x) for x in t.values()])
lines.append(line + "\n")
f.writelines(lines)
print ("All Done!")
exit()
utils.mosaic_tiles(work_path + "*_dsm.tif", work_path + "mosaic_dsm.tif", minx, miny, maxx, maxy,
None, job_id)
utils.mosaic_tiles(work_path + "*_height.tif", work_path + "mosaic_height.tif", minx, miny, maxx,
maxy, None, job_id)
# utils.mosaic_tiles(self.work_path + "*_intensity.tif", self.work_path + "mosaic_intensity.tif", minx, miny, maxx, maxy, clip_poly)
utils.mosaic_tiles(work_path + "*_dem.tif", work_path + "mosaic_dem.tif", minx, miny, maxx, maxy,
None, job_id)
tree_file = work_path + "mosaic_trees.shp"
utils.merge_tiles(work_path + "*_trees.shp", tree_file, None)
utils.finalize(tree_file, None, job_id)
bldgs_file = work_path + "mosaic_bldgs.shp"
utils.merge_tiles(work_path + "*_bldgs.shp", bldgs_file, None, 10)
utils.finalize(bldgs_file, None, job_id)
# impervious_file = self.work_path + "mosaic_impervious.shp"
# utils.merge_tiles(self.work_path + "*_impervious.shp", impervious_file, clip_poly, 10)
# utils.finalize(impervious_file, clip_poly, self.job_id)
contours_file = work_path + "mosaic_contours.shp"
utils.contours(work_path + "mosaic_dem.tif", contours_file, None)
utils.finalize(contours_file, None, job_id)
print("job complete!")
t1 = time.time()
im = imed_metric(triv_pred, example_lbls, G=G)
t2 = time.time()
trivial_error[i] = im
t3 = time.time()
tqdm.write(f"imed: {t2-t1}")
tqdm.write(f"asgn: {t3-t2}")
if animate:
anim = animate_double_imshow(example_lbls, example_pred)
plt.show()
return np.array(esn_error).mean(axis=0), np.array(trivial_error).mean(
axis=0)
if __name__ == "__main__":
from utils import initialize, finalize
args = initialize()
data_dir = pathlib.Path("/mnt/data/torsk_experiments")
outdir = data_dir / "agulhas_3daymean_50x30"
esn_error, trivial_error = esn_perf(outdir, animate=True)
fig, ax = plt.subplots(1, 1)
ax.plot(esn_error, label="ESN")
ax.plot(trivial_error, label="trivial")
ax.set_ylabel("Error")
ax.set_xlabel("Days")
finalize(args, fig, [ax], loc="upper left")
sns.set_context("notebook")
pred_data_ncfiles, indices = sort_filenames(pred_data_ncfiles,
return_indices=True)
pixel_error, trivial_error, cycle_error = [], [], []
for idx, pred_data_nc in tqdm(zip(indices, pred_data_ncfiles),
total=len(indices)):
tqdm.write(pred_data_nc.as_posix())
with nc.Dataset(pred_data_nc, "r") as src:
outputs = src["outputs"][:valid_pred_length]
labels = src["labels"][:valid_pred_length]
error_seq = np.abs(outputs - labels)
error = np.mean(error_seq, axis=0)
pixel_error.append(error)
pixel_error = np.array(pixel_error)
pixel_score, _, _, _ = sliding_score(pixel_error,
small_window=small_window,
large_window=large_window)
fig, ax = plt.subplots(1, 1)
pixel_count = np.sum(pixel_score < normality_threshold, axis=0)
im = ax.imshow(pixel_count)
plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04)
finalize(args, fig, [ax], loc="lower left")