def stats_page(): global flight_progress_stats_updated total_verified = FilmSegment.query_visible_to_user(current_user).filter(FilmSegment.is_verified == True).count() total = FilmSegment.query_visible_to_user(current_user).count() if current_user.is_authenticated: include_greenland = current_user.view_greenland else: include_greenland = False flightprogress_html = make_flight_progress_bar_plot(include_greenland=include_greenland) elapsed_time = time.time() - flight_progress_stats_updated if elapsed_time < 5: update_string = "just now" elif elapsed_time < 60: update_string = "less than a minute ago" else: update_string = f"about {int(elapsed_time / 60)} minutes ago" return render_template("stats.html", flightprogress=flightprogress_html, total_verified=total_verified, total=total, percent=int(100*total_verified/total), update_string=update_string, breadcrumbs=[('Explorer', '/'), ('Stats', url_for('main_bp.stats_page'))])
def add_next_previous(seg_dict, seg): # Add next/prev by frame order prev_frame = min(seg_dict['first_frame'], seg_dict['last_frame']) - 1 next_frame = max(seg_dict['first_frame'], seg_dict['last_frame']) + 1 next_by_frame = FilmSegment.query_visible_to_user(current_user).filter( (FilmSegment.first_frame >= next_frame) & (FilmSegment.last_frame >= next_frame) & (FilmSegment.reel == seg.reel) & (FilmSegment.scope_type == seg.scope_type)).order_by( FilmSegment.first_frame.asc()).first() prev_by_frame = FilmSegment.query_visible_to_user(current_user).filter( (FilmSegment.first_frame <= prev_frame) & (FilmSegment.last_frame <= prev_frame) & (FilmSegment.reel == seg.reel) & (FilmSegment.scope_type == seg.scope_type)).order_by( FilmSegment.first_frame.desc()).first() if prev_by_frame: seg_dict['prev_by_frame'] = prev_by_frame.id if next_by_frame: seg_dict['next_by_frame'] = next_by_frame.id # Add next/prev by CBD order prev_cbd = min(seg_dict['first_cbd'], seg_dict['last_cbd']) - 1 next_cbd = max(seg_dict['first_cbd'], seg_dict['last_cbd']) + 1 next_by_cbd = FilmSegment.query_visible_to_user(current_user).filter( (FilmSegment.first_cbd >= next_cbd) & (FilmSegment.last_cbd >= next_cbd) & (FilmSegment.flight == seg.flight) & (FilmSegment.scope_type == seg.scope_type)).order_by( FilmSegment.first_cbd.asc()).first() prev_by_cbd = FilmSegment.query_visible_to_user(current_user).filter( (FilmSegment.first_cbd <= prev_cbd) & (FilmSegment.last_cbd <= prev_cbd) & (FilmSegment.flight == seg.flight) & (FilmSegment.scope_type == seg.scope_type)).order_by( FilmSegment.first_cbd.desc()).first() if prev_by_cbd: seg_dict['prev_by_cbd'] = prev_by_cbd.id if next_by_cbd: seg_dict['next_by_cbd'] = next_by_cbd.id
def film_segment_history(id): if not has_write_permission(current_user): return None, 401 seg = FilmSegment.query_visible_to_user(current_user).filter( FilmSegment.id == id).first_or_404(id) history = [] for version in range(count_versions(seg)): history.append(seg.versions[version].changeset) return {'history': history}
def post(self, id): if not has_write_permission(current_user): return None, 401 seg = FilmSegment.query_visible_to_user(current_user).filter( FilmSegment.id == id).first_or_404(id) if not request.is_json: # only accept json formatted update requests return None, 400 if 'flight' in request.json: seg.flight = request.json['flight'] if 'first_cbd' in request.json: seg.first_cbd = request.json['first_cbd'] if 'last_cbd' in request.json: seg.last_cbd = request.json['last_cbd'] if 'scope_type' in request.json: seg.scope_type = request.json['scope_type'] if 'instrument_type' in request.json: seg.instrument_type = request.json['instrument_type'] if 'notes' in request.json: seg.notes = request.json['notes'] if 'raw_date' in request.json: if request.json['raw_date'] == '': seg.raw_date = None else: seg.raw_date = request.json['raw_date'] if 'is_junk' in request.json: seg.is_junk = (request.json['is_junk'] == "junk") if 'is_verified' in request.json: seg.is_verified = (request.json['is_verified'] == "verified") if 'needs_review' in request.json: seg.needs_review = (request.json['needs_review'] == "review") seg.updated_by = current_user.email seg.last_changed = datetime.now() db.session.commit() return segment_schema.dump(seg), 200
def query_id_results(): segment_ids = request.form.getlist('ids[]') query = FilmSegment.query_visible_to_user(current_user).filter(FilmSegment.id.in_(segment_ids)) if request.form.get('sort'): if request.form.get('sort') == 'cbd': query = query.order_by(FilmSegment.first_cbd) elif request.form.get('sort') == 'frame': query = query.order_by(FilmSegment.first_frame) segs = query.all() # Record this query (temporarily) query_log = {'full_query': [x.id for x in segs], 'timestamp': time.time()} qid = str(uuid.uuid4()) query_cache[qid] = query_log return render_template("queryresultslist.html", segments=segs, show_view_toggle=True, show_history=True, n_total_results=len(segs), stitch_preview=(len(segs) <= 10), query_id=qid, enable_tiff=app.config['ENABLE_TIFF'], paginate=False)
def radargram_jpg(id, max_height=None, crop_w_start=None, crop_w_end=None): seg = FilmSegment.query_visible_to_user(current_user).filter( FilmSegment.id == id).first() filename = seg.get_path(format='jpg') if max_height: im = load_image(filename) if max_height >= im.height: return serve_unmodified_image(filename) else: scale = max_height / im.height return serve_pil_image( im.resize((int(im.width * scale), int(im.height * scale)))) elif crop_w_start: im = load_image(filename) return serve_pil_image(im.crop(box=(0, 0, crop_w_start, im.size[1]))) elif crop_w_end: im = load_image(filename) return serve_pil_image( im.crop(box=(im.size[0] - crop_w_end, 0, im.size[0], im.size[1]))) else: return serve_unmodified_image(filename)
def get(self, id): seg = FilmSegment.query_visible_to_user(current_user).filter( FilmSegment.id == id).first_or_404(id) seg_dict = segment_schema.dump(seg) add_next_previous(seg_dict, seg) return seg_dict
def radargram_tiff(id): seg = FilmSegment.query_visible_to_user(current_user).filter( FilmSegment.id == id).first() return serve_unmodified_image(seg.get_path())
def film_segment_version(id, version): seg = FilmSegment.query_visible_to_user(current_user).filter( FilmSegment.id == id).first_or_404(id) seg_dict = segment_schema.dump(seg.versions[version]) add_next_previous(seg_dict, seg) return seg_dict
def make_cbd_plot(session, current_user, flight_id, width, height, return_plot=False, pageref=0, dataset='antarctica', flight_date=None): if flight_date is None: df = pd.read_sql(FilmSegment.query_visible_to_user(current_user, session=session).filter(FilmSegment.dataset == dataset) \ .filter(FilmSegment.flight == flight_id).statement, session.bind) else: df = pd.read_sql(FilmSegment.query_visible_to_user(current_user, session=session).filter(FilmSegment.dataset == dataset) \ .filter(and_(FilmSegment.flight == flight_id, FilmSegment.raw_date == flight_date)).statement, session.bind) # Add colormaps to plot unique_reels = df['reel'].unique() reel_map = {r: idx for idx, r in enumerate(unique_reels)} reel_cm = plt.cm.get_cmap('viridis', len(unique_reels)) df['Color by Reel'] = df['reel'].apply( lambda x: mcolors.to_hex(reel_cm.colors[reel_map[x]])) df['Color by Verified'] = df['is_verified'].apply(lambda x: app.config[ 'COLOR_ACCENT'] if x else app.config['COLOR_GRAY']) df['Color by Review'] = df['needs_review'].apply(lambda x: app.config[ 'COLOR_ACCENT'] if x else app.config['COLOR_GRAY']) df['Color by Frequency'] = df['instrument_type'].apply( lambda x: app.config['COLOR_REDWOOD'] if x == FilmSegment.RADAR_60MHZ else (app.config['COLOR_PALO_ALOT'] if x == FilmSegment.RADAR_300MHZ else app.config['COLOR_GRAY'])) # source = ColumnDataSource(df) toggle_verified = Toggle(label="Show only verified") toggle_junk = Toggle(label="Hide junk", active=True) toggle_z = Toggle(label="Show Z Scopes", active=True) toggle_a = Toggle(label="Show A Scopes", active=True) toggle_verified.js_on_change( 'active', CustomJS(args=dict(source=source), code="source.change.emit()")) toggle_junk.js_on_change( 'active', CustomJS(args=dict(source=source), code="source.change.emit()")) toggle_z.js_on_change( 'active', CustomJS(args=dict(source=source), code="source.change.emit()")) toggle_a.js_on_change( 'active', CustomJS(args=dict(source=source), code="source.change.emit()")) filter_verified = CustomJSFilter(args=dict(tog=toggle_verified), code=''' var indices = []; for (var i = 0; i < source.get_length(); i++){ if ((!tog.active) || source.data['is_verified'][i]){ indices.push(true); } else { indices.push(false); } } return indices; ''') filter_junk = CustomJSFilter(args=dict(tog=toggle_junk), code=''' var indices = []; for (var i = 0; i < source.get_length(); i++){ if (tog.active && source.data['is_junk'][i]){ indices.push(false); } else { indices.push(true); } } console.log(indices); return indices; ''') filter_scope = CustomJSFilter(args=dict(tog_a=toggle_a, tog_z=toggle_z), code=''' console.log('filter_scope'); var indices = []; for (var i = 0; i < source.get_length(); i++){ if (tog_a.active && (source.data['scope_type'][i] == 'a')){ indices.push(true); } else if (tog_z.active && (source.data['scope_type'][i] == 'z')) { indices.push(true); } else { indices.push(false); } } console.log(indices); return indices; ''') view = CDSView(source=source, filters=[filter_verified, filter_junk, filter_scope]) p = figure( tools=['pan,box_zoom,wheel_zoom,box_select,lasso_select,reset,tap'], active_scroll='wheel_zoom') segs = p.segment(y0='first_frame', y1='last_frame', x0='first_cbd', x1='last_cbd', color=app.config["COLOR_GRAY"], source=source, view=view) scat_first = p.scatter('first_cbd', 'first_frame', color='Color by Verified', source=source, view=view, nonselection_fill_color=app.config["COLOR_GRAY"]) scat_last = p.scatter('last_cbd', 'last_frame', color='Color by Verified', source=source, view=view, nonselection_fill_color=app.config["COLOR_GRAY"]) p.add_tools( HoverTool(renderers=[segs], tooltips=[("Reel", "@reel"), ("Scope", "@scope_type"), ("Verified", "@is_verified")])) p.xaxis.axis_label = "CBD" p.yaxis.axis_label = "Frame" p.sizing_mode = "stretch_both" if (width is not None) and (height is not None): p.width = width p.height = height p.title.text = "Film Segments" # Select matching code from https://stackoverflow.com/questions/54768576/python-bokeh-customjs-debugging-a-javascript-callback-for-the-taping-tool cb_cselect = CustomJS(args=dict(s1=scat_first, s2=scat_last, csource=source), code=""" var selected_color = cb_obj.value; s1.glyph.line_color.field = selected_color; s1.glyph.fill_color.field = selected_color; s2.glyph.line_color.field = selected_color; s2.glyph.fill_color.field = selected_color; csource.change.emit(); """) color_select = Select(value="Color by Verified", options=[ "Color by Verified", "Color by Reel", "Color by Review", "Color by Frequency" ]) color_select.js_on_change('value', cb_cselect) p.toolbar.active_scroll = p.select_one(WheelZoomTool) if return_plot: return p, column(toggle_verified, toggle_junk, toggle_z, toggle_a, color_select), source else: layout = row( p, column(toggle_verified, toggle_junk, toggle_z, toggle_a, color_select)) script, div = components(layout) return f'\n{script}\n\n{div}\n'
def update_page(id): seg = FilmSegment.query_visible_to_user(current_user).filter(FilmSegment.id == id).first() return render_template("update.html", segment=seg, enable_tiff=app.config['ENABLE_TIFF'], breadcrumbs=[('Explorer', '/')])
def query_results(): query = FilmSegment.query_visible_to_user(current_user) # Filters if request.args.get('flight'): query = query.filter(FilmSegment.flight == int(request.args.get('flight'))) if request.args.get('reel'): query = query.filter(FilmSegment.reel == request.args.get('reel')) if request.args.get('verified'): if int(request.args.get('verified')) == 0: query = query.filter(FilmSegment.is_verified == False) elif int(request.args.get('verified')) == 1: query = query.filter(FilmSegment.is_verified == True) if request.args.get('scope'): query = query.filter(FilmSegment.scope_type == request.args.get('scope')) if request.args.get('dataset'): query = query.filter(FilmSegment.dataset == request.args.get('dataset')) if request.args.get('mincbd'): query = query.filter(FilmSegment.first_cbd >= int(request.args.get('mincbd'))) if request.args.get('maxcbd'): query = query.filter(FilmSegment.first_cbd <= int(request.args.get('maxcbd'))) if request.args.get('minframe'): query = query.filter(FilmSegment.first_frame >= int(request.args.get('minframe'))) if request.args.get('maxframe'): query = query.filter(FilmSegment.first_frame <= int(request.args.get('maxframe'))) # Sorting if request.args.get('sort'): if request.args.get('sort') == 'cbd': query = query.order_by(FilmSegment.first_cbd) elif request.args.get('sort') == 'frame': query = query.order_by(FilmSegment.first_frame) # Number and page of results if request.args.get('n'): n = int(request.args.get('n')) else: n = 10 if request.args.get('skip'): query.offset(int(request.args.get('skip'))) n_total_results = query.count() n_pages = math.ceil(n_total_results / n) if request.args.get('page'): current_page = int(request.args.get('page')) query = query.offset((current_page-1) * n) else: current_page = 1 query_page = query.limit(n) # Just this page # Record this query (temporarily) query_log = {'full_query': [x.id for x in query.all()], 'page_query': [x.id for x in query_page.all()], 'timestamp': time.time()} qid = str(uuid.uuid4()) query_cache[qid] = query_log # Display options if request.args.get('history') and int(request.args.get('history')) == 0: show_history = 0 else: show_history = 1 # Figure out pagination # All the page links should repeat all the GET arguments except the page base_args = {k: v for k,v in request.args.items() if k != 'page'} # Create an ordered dictionary of page numbers to links to the appropriate query pages = OrderedDict() # Our convention here is to display links to the first page, the the last page, and the pages before/after the current for pg in [1, current_page-1, current_page, current_page+1, n_pages]: if (pg > 0) and (pg <= n_pages) and (not (pg in pages)): # Sometimes this list may overlap - don't repeat any pages pages[pg] = url_for('main_bp.query_results', page=pg, **base_args) prev_page = current_page - 1 # A value of 0 indicates no previous page if current_page < n_pages: next_page = current_page + 1 else: next_page = 0 # Return results segs = query_page.all() return render_template("queryresults.html", segments=segs, show_view_toggle=True, show_history=show_history, n_total_results=n_total_results, n_pages=n_pages, current_page=current_page, paginate=True, next_page=next_page, prev_page=prev_page, page_map=pages, query_id=qid, enable_tiff=app.config['ENABLE_TIFF'], breadcrumbs=[('Explorer', '/'), ('Query Results', url_for('main_bp.query_results'))])
def query_bulk_action(): t = time.time() qid = request.form['query_id'] action_type = request.form['action_type'] scope = request.form['scope'] if not (has_write_permission(current_user) or (action_type == 'stitch')): return "You're not logged in or don't have the appropriate permissions." if not qid or (not (qid in query_cache)): return "No query id specified or query id invalid. If you've had this page open more than an hour, your query "\ "may have expired. " if not action_type: return "No action specified" if not scope: return "No scope specified" query_log = query_cache[qid] if scope == 'page': query_ids = query_log['page_query'] elif scope == 'query': query_ids = query_log['full_query'] else: return "Invalid scope specified" query = FilmSegment.query_visible_to_user(current_user).filter(FilmSegment.id.in_(query_ids)) if action_type == 'mark_verified': for f in query.all(): f.is_verified = True f.updated_by = current_user.email f.last_changed = datetime.now() db.session.commit() elif action_type == 'set_60mhz': for f in query.all(): f.instrument_type = 10 f.updated_by = current_user.email f.last_changed = datetime.now() db.session.commit() elif action_type == 'set_300mhz': for f in query.all(): f.instrument_type = 20 f.updated_by = current_user.email f.last_changed = datetime.now() db.session.commit() elif action_type == 'stitch': image_type = request.form.get('format', 'jpg') scale_x = float(request.form.get('scale_x', 1)) scale_y = float(request.form.get('scale_y', 1)) flip = request.form.get('flip', "") if ((query.count() > 10) or image_type != 'jpg') and not has_write_permission(current_user): return "Must be logged in with appropriate permissions to stitch more than 10 images." if query.count() > 10 and image_type != 'jpg': return "Sorry, merging more than 10 images into TIFF format is not yet supported due to the absurd size of the original TIFF images." query = query.order_by(FilmSegment.first_cbd) img_paths = [f.get_path() for f in query.all()] job = queue.enqueue(stitch_images, failure_ttl=60, args=(img_paths, image_type, flip, scale_x, scale_y, qid)) return f"started:{job.get_id()}" else: return "Unknown action type" print(f"Request type [{action_type}] took {time.time() - t} seconds to process") return "success"