def check_job_finished(self, job_id=0, user=None): from app.jobs.jobs import get_job j = get_job(job_id) history = self.get_history(j.get_output_parameter("history_id")) if j.has_permission(user): status = j.check_status() if status == "complete": history = self.get_history( j.get_output_parameter("history_id")) vs = ViewSet(self.db, self.get_viewset_id()) c_d = {"data": [], "columns": []} if len(history["fields"]) > 0: c_d = vs.get_columns_and_data(history["fields"]) graphs = [] tracks = [] for g in self.data["graph_config"]: if g["id"] in history["graphs"]: graphs.append(g) for t in self.data["browser_config"]["state"]: if t["track_id"] in history["tracks"]: tracks.append(t) return { "status": "complete", "columns": c_d["columns"], "data": c_d["data"], "tracks": tracks, "graphs": graphs, "history": history } else: return {"status": status, "history": history}
def fix_tags_in_set(db,project_id): set_name = request.form.get("set") tag_set = request.form.get("tags") if tag_set: tag_set=ujson.loads(tag_set) p = get_project(project_id) if not p.has_edit_permission(current_user): return ujson.dumps({"success":False,"msg":"You do not have permission"}) vs =ViewSet(db,p.data['viewset_id']) set_info=None for item in p.data['ml_peak_class']['sets']: if item['name']==set_name: set_info=item break tags =vs.add_tags_to_set(set_name,set_info['label'], zegami_url=set_info.get('zegami_url'), tags=tag_set,p=p) total=0 for t in tags: total+=tags[t] set_info['tags_submitted']=True set_info['tags']=tags set_info['tags']['all']=set_info['size'] tag_field=vs.data["field_information"]['sets'][set_name][set_name+"_tags"] p.update() return ujson.dumps({"success":True,"set":set_info,"tag_field":tag_field})
def create_subset_from_parent(self, ids=[], parent_id=None): try: p = get_project(parent_id) user = db.session.query(User).filter_by(id=self.owner).one() if not p.has_view_permission(user): raise Exception( "User does not have permission on parent to create subset") #copy data self.data = p.data self.data["creating_subset"] = True self.update({"parent": parent_id}) #copy viewset vs = ViewSet(p.db, p.get_viewset_id()) new_vs = vs.clone(ids, "cloned from " + p.name) self.set_data("viewset_id", new_vs.id) #copy any specific files self.copy_files(p, new_vs) self.set_data("creating_subset", False) except Exception as e: app.logger.exception("Creating subset #{} from parent#{}".format( self.id, parent_id)) self.set_data("creating_subset", False) self.set_data("subset_creation_failed", True)
def process(self): try: p = get_project(self.job.inputs["project_id"]) vs = ViewSet(p.db, p.get_viewset_id()) data = vs.add_ts_starts(overlap_column=True, go_levels=self.job.inputs.get( "go_levels", 0)) p.refresh_data() p.data["graph_config"] += data["graphs"] hid = self.get_input_parameter("history_id") history = p.get_history(hid) history["tracks"] = [] history["fields"] = data["fields"] history["graphs"] = [] for g in data["graphs"]: history["graphs"].append(g["id"]) history["status"] = "complete" p.update() self.complete() except Exception as e: app.logger.exception( "Cannot process AnnotationIntersectionJob # {}".format( self.job.id)) p.refresh_data() hid = self.get_input_parameter("history_id") history = p.get_history(hid) history["status"] = "failed" p.update() self.failed(traceback.format_exc())
def create_new_zegami_set(db,project_id,tag,parent): p = Project(db,int(project_id)) sets = p.data['ml_peak_class']['sets'] vs = ViewSet(db,p.data['initial_peak_calls']['view_set_id']) name = "set_{}".format(len(sets)) bed_file=vs.create_new_zegami_set(tag,name) num = len(sets) set_info={ "name":name, "tags":{}, "created_from":[parent,tag] } job_data={ "bed_file":bed_file, "wig_file":p.data['local_wig_file'], "chromosome":p.data['parameter_search']['original_chromosome'], "view_set_id":vs.id, "ml_step":"False", "set_info":[p.id,name] } pcj = PeakClusteringJob(data=job_data,user_id=current_user.id,genome=db) set_info["clustering_job_id"]=pcj.job.id set_info["clustering_job"]="running" sets.append(set_info) p.update({"status":"Creating {}".format(name)}) pcj.send() return ujson.dumps({"success":True,"job_id":pcj.job.id})
def get_multi_feature_set(db, fs_id, chr, start, finish): fs_id = int(fs_id) start = int(start) finish = int(finish) fs = ViewSet(db, fs_id) metadata = fs.get_metadata() data = fs.get_views(chr, start, finish) return json.dumps({"metadata": metadata, "data": data})
def create_anno_bed_file(self, force=False): vs = ViewSet(self.db, self.get_viewset_id()) vbf = vs.get_bed_file(force) pbf = os.path.join(self.get_tracks_folder(), "anno_{}.bed.gz".format(self.id)) copyfile(vbf, pbf) copyfile(vbf + ".tbi", pbf + ".tbi") self.set_data("bed_file", pbf)
def get_bw_stats_data(self, job_id=0): from app.jobs.jobs import get_job j = get_job(job_id) wig_names = j.get_input_parameter("wig_names") vs = ViewSet(self.db, self.get_viewset_id()) data = vs.get_wig_stats_data(wig_names) wig_files = [] for track in self.data["browser_config"]["state"]: if track['track_id'] in wig_names: wig_files.append(track) data["tracks"] = wig_files return data
def cluster_by_fields(self, fields, name, methods, dimensions): vs = ViewSet(self.db, self.get_viewset_id()) info = vs.cluster_by_fields(fields, name, methods, dimensions) self.refresh_data() graphs = [] fds = [] for cols in info: gid = cols["method"] + "_" + random_string(5) self.data["graph_config"].append({ "type": "wgl_scatter_plot", "title": name + " " + cols["method"], "param": [cols["fields"][0], cols["fields"][1]], "id": gid, "axis": { "x_label": cols["labels"][0], "y_label": cols["labels"][1] }, "location": { "x": 0, "y": 0, "height": 4, "width": 6 } }) graphs.append(gid) for f in cols["fields"]: fds.append(f) field_names = [] for f in fields: field_names.append(vs.fields[f]["label"]) gid = "abc_" + random_string(5) graphs.append(gid) self.data["graph_config"].append({ "type": "average_bar_chart", "title": name + " Fields", "param": fields, "labels": field_names, "id": gid, "location": { "x": 0, "y": 0, "height": 2, "width": 4 } }) self.update() return {"graphs": graphs, "tarcks": [], "fields": fds}
def check_tag_set_clustering(db,job_id): try: job = get_job(job_id) job.check_status() if job.job.status=="complete": si = job.job.inputs['set_info'] project_id=si[0] set= si[1] p= get_project(project_id) vs_id= p.data['initial_peak_calls']['view_set_id'] vs = ViewSet(db,vs_id) fields= vs.data['field_information']["sets"][set] sql ="SELECT id,{},{},{},{},{} from {} WHERE {}='True'".format(fields[set+"_tSNE1"],fields[set+"_tSNE2"], fields[set+"_PCA1"],fields[set+"_PCA2"], fields['is_set'], vs.table_name,fields['is_set']) results = databases[db].execute_query(sql) ret_dict={} for r in results: i= r['id'] del r['id'] ret_dict[i]=r set_info=None for item in p.data['ml_peak_class']['sets']: if item["name"]==set: set_info=item return ujson.dumps({"complete":True,"data":ret_dict,"fields":fields,"set":set_info}) else: return ujson.dumps({"complete":False}) except Exception as e: app.logger.exception("cannot processs job".format(db,project_id)) return ujson.dumps({"success":False,"msg":"Could not create set"})
def process(self): try: p = get_project(self.job.inputs["project_id"]) vs = ViewSet(p.db, p.get_viewset_id()) create_thumbnails_from_ucsc(self.job.inputs["session_url"], vs, pixels=self.job.inputs["image_width"], margins=self.job.inputs.get( "margins", 0), job=self.job) p.set_data("has_images", True) #send email url = p.get_url(external=True) user = db.session.query(User).filter_by(id=self.job.user_id).one() send_email(user, "Images Created", "ucsc_image_job_finished", url=url) p.set_data("creating_images_job_status", "complete") self.complete() except Exception as e: app.logger.exception("Error in ucsc upload # {}".format( self.job.id)) p.set_data("creating_images_job_status", "failed") self.failed(traceback.format_exc())
def create_compound_column(self, stages=[], name="", final_trans="none", history={}): vs = ViewSet(self.db, self.get_viewset_id()) data = vs.create_compound_column(name, stages, final_trans) history["status"] = "complete" history["tracks"] = [] f = data["columns"][0]["field"] history["fields"] = [f] history["id"] = f history["graphs"] = [data["graphs"][0]["id"]] self.add_to_history(history) data["history"] = history return data
def create_thumbnails(db, vs_id): data = request.json max_y = None if data['primary_track']['scale'] == "fixed": max_y = data['primary_track']['max_y'] primary_color = data['primary_track']['color'] vs = ViewSet(db, vs_id) vs.create_thumbnails(25, 50, max_y=max_y, primary_track_color=primary_color, secondary_track=data.get('secondary_track'), flanking_region=data.get("margin"), preview=data.get("preview"), limit=data.get("limit")) return ujson.dumps({"success": True})
def add_tagging_column(self): vs = ViewSet(self.db, self.get_viewset_id()) l_to_f = vs.add_columns([{ "label": "Tags", "datatype": "text" }], "current_tags") field = l_to_f["Tags"] if not self.data.get("graph_config"): self.data["graph_config"] = [] self.data["graph_config"].append({ "type": "row_chart", "title": "Tags", "param": field, "id": "current_tags" }) self.update()
def delete(self, hard=False): if not hard: return super().delete() vs_id = self.data.get("viewset_id") if vs_id: vs = ViewSet(self.db, vs_id) if vs.id != -1: vs.delete(True) info = self.data.get("zegami_upload_job") if info: delete_job(info["job_id"]) info = self.data.get("ucsc_images") if info: delete_job(info["job_id"]) super().delete(True)
def get_view_set_full(db, vs_id): vs_id = int(vs_id) vs = ViewSet(db, vs_id) if request.args.get("simple"): data = request.json filters = None if data: filters = data.get("filters") view_data = vs.get_data_simple(filters=filters) else: view_data = vs.get_data_for_table() if request.args.get("project_id"): pid = int(request.args.get("project_id")) p = get_project(pid) perm = p.get_permissions(current_user) p.data["permission"] = perm view_data['project_data'] = p.data return json.dumps(view_data)
def get_cluster_data(self, job_id=0): from app.jobs.jobs import get_job j = get_job(job_id) name = j.get_input_parameter("name") methods = j.get_input_parameter("methods") vs = ViewSet(self.db, self.get_viewset_id()) data = vs.get_cluster_data(name, methods) graph_ids = [] graphs = [] field_graph = name + "_fields" for method in methods: graph_ids.append(name + "_" + method) for graph in self.data["graph_config"]: if graph['id'] in graph_ids: graphs.append(graph) if graph["id"] == field_graph: graphs.append(graph) data["graphs"] = graphs return data
def update_tags(self, tags={}, tag_color_scheme={}): vs = ViewSet(self.db, self.get_viewset_id()) ctf = vs.data["field_information"]["current_tags"]["Tags"] update_list = [] for id in tags: update_list.append({"id": id, ctf: tags[id]}) sql = "UPDATE {} SET {}=NULL".format(vs.table_name, ctf) databases[self.db].execute_update(sql) databases[self.db].update_table_with_dicts(update_list, vs.table_name) self.set_data("tag_color_scheme", tag_color_scheme)
def add_intersections(self, ids, ec): vs = ViewSet(self.db, self.get_viewset_id()) if ec: data = vs.add_annotation_fields(ids[0], ec) else: data = vs.add_annotations_intersect(ids) self.refresh_data() self.data["graph_config"] += data["graphs"] self.data["browser_config"]["state"] += data["tracks"] self.update() tracks = [] graphs = [] for t in data["tracks"]: tracks.append(t["track_id"]) for g in data["graphs"]: graphs.append(g["id"]) return {"graphs": graphs, "tracks": tracks, "fields": data["fields"]}
def add_bw_stats(self, url, name): #copy wigfile file_name = os.path.split(url)[1] folder = os.path.join(app.config['TRACKS_FOLDER'], "projects", str(self.id)) if not os.path.exists(folder): os.makedirs(folder) local_file = os.path.join(folder, file_name) url = shlex.quote(url) command = "curl {} -o {}".format(url, local_file) os.system(command) self.refresh_data() tid = random_string(5) self.data["browser_config"]["state"].append({ "url": local_file.replace("/data", ""), "track_id": tid, "discrete": True, "height": 100, "color": "#808080", "scale": "dynamic", "short_label": name, "type": "bigwig", "allow_user_remove": True, "format": "wig" }) self.update() vs = ViewSet(self.db, self.get_viewset_id()) fields = vs.add_wig_stats(local_file, name) return {"fields": fields, "track": tid}
def create_thumbnail_preview(db): data = request.json vs = ViewSet(db, data["viewset_id"]) folder = get_temporary_folder() tn_det = data['thumbnail_details'] vs.data['thumbnail_details'] = tn_det vs.data["margin"] = tn_det['margin'] vs.data["annotation_sets"] = data["annotations"] tn = ThumbnailSet(db, height=int(tn_det['height']), width=int(tn_det['width'])) ids = list(range(1, 21)) gene_set = 0 if data.get("genes") and vs.db != "other": gene_set = 1 tn.draw_view_set(vs, specific_views=ids, folder=folder, gene_set=gene_set) images = [] url_folder = folder.replace("mlv", "") for i_id in ids: images.append("{}/tn{}.png".format(url_folder, i_id)) return ujson.dumps({"images": images, "success": True})
def process(self, pword): try: p = get_project(self.job.inputs["project_id"]) p.set_data("zegami_upload_job", { "job_id": self.job.id, "job_status": "running" }) vs = ViewSet(p.db, p.get_viewset_id()) credentials = { "project": self.job.inputs["project"], "username": self.job.inputs["username"], "password": pword } url = vs.create_zegami_collection(name=p.name, job=self.job, credentials=credentials) p.set_data("zegami_url", url) vs.upload_images_to_zegami(job=self.job, credentials=credentials) #send email user = db.session.query(User).filter_by(id=self.job.user_id).one() send_email(user, "Zegami Collection Created", "zegami_collection_created", url=url) self.complete() except Exception as e: app.logger.exception("Error in zegami upload # {}".format( self.job.id)) p.set_data("zegami_upload_job", { "job_id": self.job.id, "job_status": "failed" }) self.failed(traceback.format_exc())
def create_from_project(self, project_id=None, ids=[], fields=[]): p = get_project(project_id) vs = ViewSet(p.db, p.get_viewset_id()) new_fields = [] for count, field in enumerate(fields, start=1): info = vs.fields.get(field) if not info: continue else: new_fields.append({ "datatype": info["datatype"], "position": count, "field": "f{}".format(count), "name": info["label"] }) bed = vs.create_basic_bed_file(without_ids=True, selected_ids=ids, fields=fields) files = {"upload_file": bed} print(bed) self.create_from_file(files, fields=new_fields, process_file=False)
def delete_columns(self, columns=[]): vs = ViewSet(self.db, self.get_viewset_id()) vs.remove_columns(columns) self.refresh_data() new_graphs = [] for g in self.data["graph_config"]: param = g.get("param") if param: contains_field = False if isinstance(param, list): for p in param: if p in columns: contains_field = True break else: if param in columns: contains_field = True if contains_field: continue new_graphs.append(g) if len(self.data["graph_config"]) != len(new_graphs): self.data["graph_config"] = new_graphs self.update() history = self.data.get("history") if history: new_history = [] for h in history: if h["id"] in columns: continue new_history.append(h) self.set_data("history", new_history) return self.data["history"]
def label_history(self, history): self.set_input_parameter("history_id", history["id"]) history["label"] = "Dimension Reduction " + self.job.inputs["name"] history["job_id"] = self.job.id info = "" p = get_project(self.job.inputs["project_id"]) vs = ViewSet(p.db, p.get_viewset_id()) names = [] for f in self.job.inputs["fields"]: names.append(vs.fields[f]["label"]) info += "Fields: " + ", ".join(names) info += " \nCluster Methods:" + ", ".join(self.job.inputs["methods"]) info += " \nDimensions: " + str(self.job.inputs["dimensions"]) history["info"] = info
def get_fields(self): vsid = self.get_viewset_id() if vsid: ret_list = [] vs = ViewSet(self.db, vsid) for field in vs.fields: item = vs.fields[field] item["name"] = item["label"] del item["label"] item["field"] = field ret_list.append(item) return ret_list else: return []
def remove_annotation_intersections(self, ids): vs = ViewSet(self.db, self.get_viewset_id()) info = vs.data["annotation_information"] for aid in ids: anno = info.get(str(aid)) if not anno: continue del vs.data["field_information"]["Annotations"][anno["label"]] del vs.data["annotation_information"][aid] vs.remove_columns([anno['field']]) vs.update()
def process(self): try: p = get_project(self.job.inputs["project_id"]) vs = ViewSet(p.db, p.get_viewset_id()) create_thumbnails_from_mlv( self.job.inputs["tracks"], vs, image_width=self.job.inputs["image_width"], margins=self.job.inputs.get("margins", 0), job=self.job) p.refresh_data() p.set_data("has_images", True) hid = self.get_input_parameter("history_id") history = p.get_history(hid) history["tracks"] = [] history["fields"] = [] history["graphs"] = [] history["status"] = "complete" p.update() #send email url = p.get_url(external=True) user = db.session.query(User).filter_by(id=self.job.user_id).one() send_email(user, "Images Created", "ucsc_image_job_finished", url=url) self.complete() except Exception as e: p = get_project(self.job.inputs["project_id"]) hid = self.get_input_parameter("history_id") history = p.get_history(hid) history["status"] = "failed" p.update() app.logger.exception("Error in creating mlv images # {}".format( self.job.id)) self.failed(traceback.format_exc())
def delete_tss_distances(self): vs = ViewSet(self.db, self.get_viewset_id()) fields = vs.data["field_information"].get("TSS").values() vs.remove_columns(fields) del vs.data["field_information"]["TSS"] vs.update() new_charts = [] for chart in self.data["graph_config"]: if chart['id'].startswith("_tss"): continue new_charts.append(chart) self.data["graph_config"] = new_charts del self.data["find_tss_distances_job_id"] del self.data["find_tss_distances_job_status"] self.update()
def get_viewset(self, filters=None, offset=None, limit=None): vs = ViewSet(self.db, self.get_viewset_id()) if (offset or offset == 0) and limit: off_lim = [int(offset), int(limit)] if offset == 0: return { "views": vs.get_all_views(filters=filters, offset=off_lim), "field_information": vs.data["field_information"], "fields": vs.fields, "sprite_sheets": vs.data.get("sprite_sheets"), "annotation_information": vs.data.get("annotation_information"), "base_image_url": "/data/{}/view_sets/{}/thumbnails/tn".format( self.db, vs.id), "total": vs.get_view_number()["count"] } else: return vs.get_all_views(filters=filters, offset=off_lim) else: return { "views": vs.get_all_views(filters=filters), "field_information": vs.data["field_information"], "fields": vs.fields, "sprite_sheets": vs.data.get("sprite_sheets"), "annotation_information": vs.data.get("annotation_information"), "base_image_url": "/data/{}/view_sets/{}/thumbnails/tn".format(self.db, vs.id) }