def stats_page():
global flight_progress_stats_updated
total_verified = FilmSegment.query_visible_to_user(current_user).filter(FilmSegment.is_verified == True).count()
total = FilmSegment.query_visible_to_user(current_user).count()
if current_user.is_authenticated:
include_greenland = current_user.view_greenland
else:
include_greenland = False
flightprogress_html = make_flight_progress_bar_plot(include_greenland=include_greenland)
elapsed_time = time.time() - flight_progress_stats_updated
if elapsed_time < 5:
update_string = "just now"
elif elapsed_time < 60:
update_string = "less than a minute ago"
else:
update_string = f"about {int(elapsed_time / 60)} minutes ago"
return render_template("stats.html", flightprogress=flightprogress_html,
total_verified=total_verified, total=total, percent=int(100*total_verified/total),
update_string=update_string,
breadcrumbs=[('Explorer', '/'), ('Stats', url_for('main_bp.stats_page'))])
def add_next_previous(seg_dict, seg):
# Add next/prev by frame order
prev_frame = min(seg_dict['first_frame'], seg_dict['last_frame']) - 1
next_frame = max(seg_dict['first_frame'], seg_dict['last_frame']) + 1
next_by_frame = FilmSegment.query_visible_to_user(current_user).filter(
(FilmSegment.first_frame >= next_frame)
& (FilmSegment.last_frame >= next_frame)
& (FilmSegment.reel == seg.reel)
& (FilmSegment.scope_type == seg.scope_type)).order_by(
FilmSegment.first_frame.asc()).first()
prev_by_frame = FilmSegment.query_visible_to_user(current_user).filter(
(FilmSegment.first_frame <= prev_frame)
& (FilmSegment.last_frame <= prev_frame)
& (FilmSegment.reel == seg.reel)
& (FilmSegment.scope_type == seg.scope_type)).order_by(
FilmSegment.first_frame.desc()).first()
if prev_by_frame:
seg_dict['prev_by_frame'] = prev_by_frame.id
if next_by_frame:
seg_dict['next_by_frame'] = next_by_frame.id
# Add next/prev by CBD order
prev_cbd = min(seg_dict['first_cbd'], seg_dict['last_cbd']) - 1
next_cbd = max(seg_dict['first_cbd'], seg_dict['last_cbd']) + 1
next_by_cbd = FilmSegment.query_visible_to_user(current_user).filter(
(FilmSegment.first_cbd >= next_cbd)
& (FilmSegment.last_cbd >= next_cbd)
& (FilmSegment.flight == seg.flight)
& (FilmSegment.scope_type == seg.scope_type)).order_by(
FilmSegment.first_cbd.asc()).first()
prev_by_cbd = FilmSegment.query_visible_to_user(current_user).filter(
(FilmSegment.first_cbd <= prev_cbd)
& (FilmSegment.last_cbd <= prev_cbd)
& (FilmSegment.flight == seg.flight)
& (FilmSegment.scope_type == seg.scope_type)).order_by(
FilmSegment.first_cbd.desc()).first()
if prev_by_cbd:
seg_dict['prev_by_cbd'] = prev_by_cbd.id
if next_by_cbd:
seg_dict['next_by_cbd'] = next_by_cbd.id
def film_segment_history(id):
if not has_write_permission(current_user):
return None, 401
seg = FilmSegment.query_visible_to_user(current_user).filter(
FilmSegment.id == id).first_or_404(id)
history = []
for version in range(count_versions(seg)):
history.append(seg.versions[version].changeset)
return {'history': history}
def post(self, id):
if not has_write_permission(current_user):
return None, 401
seg = FilmSegment.query_visible_to_user(current_user).filter(
FilmSegment.id == id).first_or_404(id)
if not request.is_json: # only accept json formatted update requests
return None, 400
if 'flight' in request.json:
seg.flight = request.json['flight']
if 'first_cbd' in request.json:
seg.first_cbd = request.json['first_cbd']
if 'last_cbd' in request.json:
seg.last_cbd = request.json['last_cbd']
if 'scope_type' in request.json:
seg.scope_type = request.json['scope_type']
if 'instrument_type' in request.json:
seg.instrument_type = request.json['instrument_type']
if 'notes' in request.json:
seg.notes = request.json['notes']
if 'raw_date' in request.json:
if request.json['raw_date'] == '':
seg.raw_date = None
else:
seg.raw_date = request.json['raw_date']
if 'is_junk' in request.json:
seg.is_junk = (request.json['is_junk'] == "junk")
if 'is_verified' in request.json:
seg.is_verified = (request.json['is_verified'] == "verified")
if 'needs_review' in request.json:
seg.needs_review = (request.json['needs_review'] == "review")
seg.updated_by = current_user.email
seg.last_changed = datetime.now()
db.session.commit()
return segment_schema.dump(seg), 200
def query_id_results():
segment_ids = request.form.getlist('ids[]')
query = FilmSegment.query_visible_to_user(current_user).filter(FilmSegment.id.in_(segment_ids))
if request.form.get('sort'):
if request.form.get('sort') == 'cbd':
query = query.order_by(FilmSegment.first_cbd)
elif request.form.get('sort') == 'frame':
query = query.order_by(FilmSegment.first_frame)
segs = query.all()
# Record this query (temporarily)
query_log = {'full_query': [x.id for x in segs],
'timestamp': time.time()}
qid = str(uuid.uuid4())
query_cache[qid] = query_log
return render_template("queryresultslist.html", segments=segs, show_view_toggle=True, show_history=True,
n_total_results=len(segs), stitch_preview=(len(segs) <= 10),
query_id=qid, enable_tiff=app.config['ENABLE_TIFF'], paginate=False)
def radargram_jpg(id, max_height=None, crop_w_start=None, crop_w_end=None):
seg = FilmSegment.query_visible_to_user(current_user).filter(
FilmSegment.id == id).first()
filename = seg.get_path(format='jpg')
if max_height:
im = load_image(filename)
if max_height >= im.height:
return serve_unmodified_image(filename)
else:
scale = max_height / im.height
return serve_pil_image(
im.resize((int(im.width * scale), int(im.height * scale))))
elif crop_w_start:
im = load_image(filename)
return serve_pil_image(im.crop(box=(0, 0, crop_w_start, im.size[1])))
elif crop_w_end:
im = load_image(filename)
return serve_pil_image(
im.crop(box=(im.size[0] - crop_w_end, 0, im.size[0], im.size[1])))
else:
return serve_unmodified_image(filename)
def get(self, id):
seg = FilmSegment.query_visible_to_user(current_user).filter(
FilmSegment.id == id).first_or_404(id)
seg_dict = segment_schema.dump(seg)
add_next_previous(seg_dict, seg)
return seg_dict
def radargram_tiff(id):
seg = FilmSegment.query_visible_to_user(current_user).filter(
FilmSegment.id == id).first()
return serve_unmodified_image(seg.get_path())
def film_segment_version(id, version):
seg = FilmSegment.query_visible_to_user(current_user).filter(
FilmSegment.id == id).first_or_404(id)
seg_dict = segment_schema.dump(seg.versions[version])
add_next_previous(seg_dict, seg)
return seg_dict
def make_cbd_plot(session,
current_user,
flight_id,
width,
height,
return_plot=False,
pageref=0,
dataset='antarctica',
flight_date=None):
if flight_date is None:
df = pd.read_sql(FilmSegment.query_visible_to_user(current_user, session=session).filter(FilmSegment.dataset == dataset) \
.filter(FilmSegment.flight == flight_id).statement, session.bind)
else:
df = pd.read_sql(FilmSegment.query_visible_to_user(current_user, session=session).filter(FilmSegment.dataset == dataset) \
.filter(and_(FilmSegment.flight == flight_id, FilmSegment.raw_date == flight_date)).statement, session.bind)
# Add colormaps to plot
unique_reels = df['reel'].unique()
reel_map = {r: idx for idx, r in enumerate(unique_reels)}
reel_cm = plt.cm.get_cmap('viridis', len(unique_reels))
df['Color by Reel'] = df['reel'].apply(
lambda x: mcolors.to_hex(reel_cm.colors[reel_map[x]]))
df['Color by Verified'] = df['is_verified'].apply(lambda x: app.config[
'COLOR_ACCENT'] if x else app.config['COLOR_GRAY'])
df['Color by Review'] = df['needs_review'].apply(lambda x: app.config[
'COLOR_ACCENT'] if x else app.config['COLOR_GRAY'])
df['Color by Frequency'] = df['instrument_type'].apply(
lambda x: app.config['COLOR_REDWOOD'] if x == FilmSegment.RADAR_60MHZ
else (app.config['COLOR_PALO_ALOT']
if x == FilmSegment.RADAR_300MHZ else app.config['COLOR_GRAY']))
#
source = ColumnDataSource(df)
toggle_verified = Toggle(label="Show only verified")
toggle_junk = Toggle(label="Hide junk", active=True)
toggle_z = Toggle(label="Show Z Scopes", active=True)
toggle_a = Toggle(label="Show A Scopes", active=True)
toggle_verified.js_on_change(
'active',
CustomJS(args=dict(source=source), code="source.change.emit()"))
toggle_junk.js_on_change(
'active',
CustomJS(args=dict(source=source), code="source.change.emit()"))
toggle_z.js_on_change(
'active',
CustomJS(args=dict(source=source), code="source.change.emit()"))
toggle_a.js_on_change(
'active',
CustomJS(args=dict(source=source), code="source.change.emit()"))
filter_verified = CustomJSFilter(args=dict(tog=toggle_verified),
code='''
var indices = [];
for (var i = 0; i < source.get_length(); i++){
if ((!tog.active) || source.data['is_verified'][i]){
indices.push(true);
} else {
indices.push(false);
}
}
return indices;
''')
filter_junk = CustomJSFilter(args=dict(tog=toggle_junk),
code='''
var indices = [];
for (var i = 0; i < source.get_length(); i++){
if (tog.active && source.data['is_junk'][i]){
indices.push(false);
} else {
indices.push(true);
}
}
console.log(indices);
return indices;
''')
filter_scope = CustomJSFilter(args=dict(tog_a=toggle_a, tog_z=toggle_z),
code='''
console.log('filter_scope');
var indices = [];
for (var i = 0; i < source.get_length(); i++){
if (tog_a.active && (source.data['scope_type'][i] == 'a')){
indices.push(true);
} else if (tog_z.active && (source.data['scope_type'][i] == 'z')) {
indices.push(true);
} else {
indices.push(false);
}
}
console.log(indices);
return indices;
''')
view = CDSView(source=source,
filters=[filter_verified, filter_junk, filter_scope])
p = figure(
tools=['pan,box_zoom,wheel_zoom,box_select,lasso_select,reset,tap'],
active_scroll='wheel_zoom')
segs = p.segment(y0='first_frame',
y1='last_frame',
x0='first_cbd',
x1='last_cbd',
color=app.config["COLOR_GRAY"],
source=source,
view=view)
scat_first = p.scatter('first_cbd',
'first_frame',
color='Color by Verified',
source=source,
view=view,
nonselection_fill_color=app.config["COLOR_GRAY"])
scat_last = p.scatter('last_cbd',
'last_frame',
color='Color by Verified',
source=source,
view=view,
nonselection_fill_color=app.config["COLOR_GRAY"])
p.add_tools(
HoverTool(renderers=[segs],
tooltips=[("Reel", "@reel"), ("Scope", "@scope_type"),
("Verified", "@is_verified")]))
p.xaxis.axis_label = "CBD"
p.yaxis.axis_label = "Frame"
p.sizing_mode = "stretch_both"
if (width is not None) and (height is not None):
p.width = width
p.height = height
p.title.text = "Film Segments"
# Select matching code from https://stackoverflow.com/questions/54768576/python-bokeh-customjs-debugging-a-javascript-callback-for-the-taping-tool
cb_cselect = CustomJS(args=dict(s1=scat_first,
s2=scat_last,
csource=source),
code="""
var selected_color = cb_obj.value;
s1.glyph.line_color.field = selected_color;
s1.glyph.fill_color.field = selected_color;
s2.glyph.line_color.field = selected_color;
s2.glyph.fill_color.field = selected_color;
csource.change.emit();
""")
color_select = Select(value="Color by Verified",
options=[
"Color by Verified", "Color by Reel",
"Color by Review", "Color by Frequency"
])
color_select.js_on_change('value', cb_cselect)
p.toolbar.active_scroll = p.select_one(WheelZoomTool)
if return_plot:
return p, column(toggle_verified, toggle_junk, toggle_z, toggle_a,
color_select), source
else:
layout = row(
p,
column(toggle_verified, toggle_junk, toggle_z, toggle_a,
color_select))
script, div = components(layout)
return f'\n{script}\n\n{div}\n'
def add_from_csv(input_file_path, updated_by_name, dataset, update_mode=True):
add_count = 0
update_count = 0
df = pd.read_csv(input_file_path, low_memory=False)
df = df[df.complete == True]
for idx, r in df.iterrows():
q = FilmSegment.query.filter(FilmSegment.path == r['relpath'])
if q.count() == 0: # This path doesn't exist in the db yet
add_count = add_count + 1
seg = FilmSegment(path=r['relpath'],
dataset=dataset,
is_junk=False,
is_verified=False,
needs_review=False,
instrument_type=FilmSegment.UNKNOWN,
notes="")
db.session.add(seg)
else:
seg = q.first()
if seg.dataset != dataset:
print(seg)
raise (Exception(
f"Matched with path from a different dataset (processing dataset {dataset})"
))
if q.count() > 1:
print(q)
print(q.count())
print(q.first())
raise (Exception(f"Multiple matches for path {r['relpath']}"))
updated = False
if seg.reel is None:
seg.reel = r['reel']
updated = True
if seg.first_frame is None and (not np.isnan(r['first_frame'])):
seg.first_frame = r['first_frame']
updated = True
if seg.last_frame is None and (not np.isnan(r['last_frame'])):
seg.last_frame = r['last_frame']
updated = True
if seg.first_cbd is None and (not np.isnan(r['first_cdb'])):
seg.first_cbd = r['first_cdb']
updated = True
if seg.last_cbd is None and (not np.isnan(r['last_cdb'])):
seg.last_cbd = r['last_cdb']
updated = True
if seg.flight is None and (not np.isnan(r['first_flight'])):
seg.flight = r['first_flight']
updated = True
if seg.raw_date is None and (not np.isnan(r['first_date'])):
seg.raw_date = r['first_date']
updated = True
if seg.raw_time is None and (not np.isnan(r['first_time'])):
seg.raw_time = r['first_time']
updated = True
if seg.raw_mode is None and (not np.isnan(r['first_mode'])):
seg.raw_mode = r['first_mode']
updated = True
if seg.scope_type is None:
seg.scope_type = r['type']
updated = True
if updated:
seg.updated_by = updated_by_name
seg.last_changed = datetime.now()
update_count = update_count + 1
db.session.commit()
print(
f"Added {add_count} and updated {update_count} to dataset {dataset} from {input_file_path}"
)
return add_count, update_count
def update_page(id):
seg = FilmSegment.query_visible_to_user(current_user).filter(FilmSegment.id == id).first()
return render_template("update.html", segment=seg, enable_tiff=app.config['ENABLE_TIFF'],
breadcrumbs=[('Explorer', '/')])
def query_results():
query = FilmSegment.query_visible_to_user(current_user)
# Filters
if request.args.get('flight'):
query = query.filter(FilmSegment.flight == int(request.args.get('flight')))
if request.args.get('reel'):
query = query.filter(FilmSegment.reel == request.args.get('reel'))
if request.args.get('verified'):
if int(request.args.get('verified')) == 0:
query = query.filter(FilmSegment.is_verified == False)
elif int(request.args.get('verified')) == 1:
query = query.filter(FilmSegment.is_verified == True)
if request.args.get('scope'):
query = query.filter(FilmSegment.scope_type == request.args.get('scope'))
if request.args.get('dataset'):
query = query.filter(FilmSegment.dataset == request.args.get('dataset'))
if request.args.get('mincbd'):
query = query.filter(FilmSegment.first_cbd >= int(request.args.get('mincbd')))
if request.args.get('maxcbd'):
query = query.filter(FilmSegment.first_cbd <= int(request.args.get('maxcbd')))
if request.args.get('minframe'):
query = query.filter(FilmSegment.first_frame >= int(request.args.get('minframe')))
if request.args.get('maxframe'):
query = query.filter(FilmSegment.first_frame <= int(request.args.get('maxframe')))
# Sorting
if request.args.get('sort'):
if request.args.get('sort') == 'cbd':
query = query.order_by(FilmSegment.first_cbd)
elif request.args.get('sort') == 'frame':
query = query.order_by(FilmSegment.first_frame)
# Number and page of results
if request.args.get('n'):
n = int(request.args.get('n'))
else:
n = 10
if request.args.get('skip'):
query.offset(int(request.args.get('skip')))
n_total_results = query.count()
n_pages = math.ceil(n_total_results / n)
if request.args.get('page'):
current_page = int(request.args.get('page'))
query = query.offset((current_page-1) * n)
else:
current_page = 1
query_page = query.limit(n) # Just this page
# Record this query (temporarily)
query_log = {'full_query': [x.id for x in query.all()],
'page_query': [x.id for x in query_page.all()],
'timestamp': time.time()}
qid = str(uuid.uuid4())
query_cache[qid] = query_log
# Display options
if request.args.get('history') and int(request.args.get('history')) == 0:
show_history = 0
else:
show_history = 1
# Figure out pagination
# All the page links should repeat all the GET arguments except the page
base_args = {k: v for k,v in request.args.items() if k != 'page'}
# Create an ordered dictionary of page numbers to links to the appropriate query
pages = OrderedDict()
# Our convention here is to display links to the first page, the the last page, and the pages before/after the current
for pg in [1, current_page-1, current_page, current_page+1, n_pages]:
if (pg > 0) and (pg <= n_pages) and (not (pg in pages)): # Sometimes this list may overlap - don't repeat any pages
pages[pg] = url_for('main_bp.query_results', page=pg, **base_args)
prev_page = current_page - 1 # A value of 0 indicates no previous page
if current_page < n_pages:
next_page = current_page + 1
else:
next_page = 0
# Return results
segs = query_page.all()
return render_template("queryresults.html", segments=segs, show_view_toggle=True, show_history=show_history,
n_total_results=n_total_results, n_pages=n_pages, current_page=current_page, paginate=True,
next_page=next_page, prev_page=prev_page, page_map=pages, query_id=qid,
enable_tiff=app.config['ENABLE_TIFF'],
breadcrumbs=[('Explorer', '/'), ('Query Results', url_for('main_bp.query_results'))])
def query_bulk_action():
t = time.time()
qid = request.form['query_id']
action_type = request.form['action_type']
scope = request.form['scope']
if not (has_write_permission(current_user) or (action_type == 'stitch')):
return "You're not logged in or don't have the appropriate permissions."
if not qid or (not (qid in query_cache)):
return "No query id specified or query id invalid. If you've had this page open more than an hour, your query "\
"may have expired. "
if not action_type:
return "No action specified"
if not scope:
return "No scope specified"
query_log = query_cache[qid]
if scope == 'page':
query_ids = query_log['page_query']
elif scope == 'query':
query_ids = query_log['full_query']
else:
return "Invalid scope specified"
query = FilmSegment.query_visible_to_user(current_user).filter(FilmSegment.id.in_(query_ids))
if action_type == 'mark_verified':
for f in query.all():
f.is_verified = True
f.updated_by = current_user.email
f.last_changed = datetime.now()
db.session.commit()
elif action_type == 'set_60mhz':
for f in query.all():
f.instrument_type = 10
f.updated_by = current_user.email
f.last_changed = datetime.now()
db.session.commit()
elif action_type == 'set_300mhz':
for f in query.all():
f.instrument_type = 20
f.updated_by = current_user.email
f.last_changed = datetime.now()
db.session.commit()
elif action_type == 'stitch':
image_type = request.form.get('format', 'jpg')
scale_x = float(request.form.get('scale_x', 1))
scale_y = float(request.form.get('scale_y', 1))
flip = request.form.get('flip', "")
if ((query.count() > 10) or image_type != 'jpg') and not has_write_permission(current_user):
return "Must be logged in with appropriate permissions to stitch more than 10 images."
if query.count() > 10 and image_type != 'jpg':
return "Sorry, merging more than 10 images into TIFF format is not yet supported due to the absurd size of the original TIFF images."
query = query.order_by(FilmSegment.first_cbd)
img_paths = [f.get_path() for f in query.all()]
job = queue.enqueue(stitch_images, failure_ttl=60, args=(img_paths, image_type, flip, scale_x, scale_y, qid))
return f"started:{job.get_id()}"
else:
return "Unknown action type"
print(f"Request type [{action_type}] took {time.time() - t} seconds to process")
return "success"