def check_job_finished(self, job_id=0, user=None):
from app.jobs.jobs import get_job
j = get_job(job_id)
history = self.get_history(j.get_output_parameter("history_id"))
if j.has_permission(user):
status = j.check_status()
if status == "complete":
history = self.get_history(
j.get_output_parameter("history_id"))
vs = ViewSet(self.db, self.get_viewset_id())
c_d = {"data": [], "columns": []}
if len(history["fields"]) > 0:
c_d = vs.get_columns_and_data(history["fields"])
graphs = []
tracks = []
for g in self.data["graph_config"]:
if g["id"] in history["graphs"]:
graphs.append(g)
for t in self.data["browser_config"]["state"]:
if t["track_id"] in history["tracks"]:
tracks.append(t)
return {
"status": "complete",
"columns": c_d["columns"],
"data": c_d["data"],
"tracks": tracks,
"graphs": graphs,
"history": history
}
else:
return {"status": status, "history": history}
def fix_tags_in_set(db,project_id):
set_name = request.form.get("set")
tag_set = request.form.get("tags")
if tag_set:
tag_set=ujson.loads(tag_set)
p = get_project(project_id)
if not p.has_edit_permission(current_user):
return ujson.dumps({"success":False,"msg":"You do not have permission"})
vs =ViewSet(db,p.data['viewset_id'])
set_info=None
for item in p.data['ml_peak_class']['sets']:
if item['name']==set_name:
set_info=item
break
tags =vs.add_tags_to_set(set_name,set_info['label'],
zegami_url=set_info.get('zegami_url'),
tags=tag_set,p=p)
total=0
for t in tags:
total+=tags[t]
set_info['tags_submitted']=True
set_info['tags']=tags
set_info['tags']['all']=set_info['size']
tag_field=vs.data["field_information"]['sets'][set_name][set_name+"_tags"]
p.update()
return ujson.dumps({"success":True,"set":set_info,"tag_field":tag_field})
def create_subset_from_parent(self, ids=[], parent_id=None):
try:
p = get_project(parent_id)
user = db.session.query(User).filter_by(id=self.owner).one()
if not p.has_view_permission(user):
raise Exception(
"User does not have permission on parent to create subset")
#copy data
self.data = p.data
self.data["creating_subset"] = True
self.update({"parent": parent_id})
#copy viewset
vs = ViewSet(p.db, p.get_viewset_id())
new_vs = vs.clone(ids, "cloned from " + p.name)
self.set_data("viewset_id", new_vs.id)
#copy any specific files
self.copy_files(p, new_vs)
self.set_data("creating_subset", False)
except Exception as e:
app.logger.exception("Creating subset #{} from parent#{}".format(
self.id, parent_id))
self.set_data("creating_subset", False)
self.set_data("subset_creation_failed", True)
def process(self):
try:
p = get_project(self.job.inputs["project_id"])
vs = ViewSet(p.db, p.get_viewset_id())
data = vs.add_ts_starts(overlap_column=True,
go_levels=self.job.inputs.get(
"go_levels", 0))
p.refresh_data()
p.data["graph_config"] += data["graphs"]
hid = self.get_input_parameter("history_id")
history = p.get_history(hid)
history["tracks"] = []
history["fields"] = data["fields"]
history["graphs"] = []
for g in data["graphs"]:
history["graphs"].append(g["id"])
history["status"] = "complete"
p.update()
self.complete()
except Exception as e:
app.logger.exception(
"Cannot process AnnotationIntersectionJob # {}".format(
self.job.id))
p.refresh_data()
hid = self.get_input_parameter("history_id")
history = p.get_history(hid)
history["status"] = "failed"
p.update()
self.failed(traceback.format_exc())
def create_new_zegami_set(db,project_id,tag,parent):
p = Project(db,int(project_id))
sets = p.data['ml_peak_class']['sets']
vs = ViewSet(db,p.data['initial_peak_calls']['view_set_id'])
name = "set_{}".format(len(sets))
bed_file=vs.create_new_zegami_set(tag,name)
num = len(sets)
set_info={
"name":name,
"tags":{},
"created_from":[parent,tag]
}
job_data={
"bed_file":bed_file,
"wig_file":p.data['local_wig_file'],
"chromosome":p.data['parameter_search']['original_chromosome'],
"view_set_id":vs.id,
"ml_step":"False",
"set_info":[p.id,name]
}
pcj = PeakClusteringJob(data=job_data,user_id=current_user.id,genome=db)
set_info["clustering_job_id"]=pcj.job.id
set_info["clustering_job"]="running"
sets.append(set_info)
p.update({"status":"Creating {}".format(name)})
pcj.send()
return ujson.dumps({"success":True,"job_id":pcj.job.id})
def get_multi_feature_set(db, fs_id, chr, start, finish):
fs_id = int(fs_id)
start = int(start)
finish = int(finish)
fs = ViewSet(db, fs_id)
metadata = fs.get_metadata()
data = fs.get_views(chr, start, finish)
return json.dumps({"metadata": metadata, "data": data})
def create_anno_bed_file(self, force=False):
vs = ViewSet(self.db, self.get_viewset_id())
vbf = vs.get_bed_file(force)
pbf = os.path.join(self.get_tracks_folder(),
"anno_{}.bed.gz".format(self.id))
copyfile(vbf, pbf)
copyfile(vbf + ".tbi", pbf + ".tbi")
self.set_data("bed_file", pbf)
def get_bw_stats_data(self, job_id=0):
from app.jobs.jobs import get_job
j = get_job(job_id)
wig_names = j.get_input_parameter("wig_names")
vs = ViewSet(self.db, self.get_viewset_id())
data = vs.get_wig_stats_data(wig_names)
wig_files = []
for track in self.data["browser_config"]["state"]:
if track['track_id'] in wig_names:
wig_files.append(track)
data["tracks"] = wig_files
return data
def cluster_by_fields(self, fields, name, methods, dimensions):
vs = ViewSet(self.db, self.get_viewset_id())
info = vs.cluster_by_fields(fields, name, methods, dimensions)
self.refresh_data()
graphs = []
fds = []
for cols in info:
gid = cols["method"] + "_" + random_string(5)
self.data["graph_config"].append({
"type":
"wgl_scatter_plot",
"title":
name + " " + cols["method"],
"param": [cols["fields"][0], cols["fields"][1]],
"id":
gid,
"axis": {
"x_label": cols["labels"][0],
"y_label": cols["labels"][1]
},
"location": {
"x": 0,
"y": 0,
"height": 4,
"width": 6
}
})
graphs.append(gid)
for f in cols["fields"]:
fds.append(f)
field_names = []
for f in fields:
field_names.append(vs.fields[f]["label"])
gid = "abc_" + random_string(5)
graphs.append(gid)
self.data["graph_config"].append({
"type": "average_bar_chart",
"title": name + " Fields",
"param": fields,
"labels": field_names,
"id": gid,
"location": {
"x": 0,
"y": 0,
"height": 2,
"width": 4
}
})
self.update()
return {"graphs": graphs, "tarcks": [], "fields": fds}
def check_tag_set_clustering(db,job_id):
try:
job = get_job(job_id)
job.check_status()
if job.job.status=="complete":
si = job.job.inputs['set_info']
project_id=si[0]
set= si[1]
p= get_project(project_id)
vs_id= p.data['initial_peak_calls']['view_set_id']
vs = ViewSet(db,vs_id)
fields= vs.data['field_information']["sets"][set]
sql ="SELECT id,{},{},{},{},{} from {} WHERE {}='True'".format(fields[set+"_tSNE1"],fields[set+"_tSNE2"],
fields[set+"_PCA1"],fields[set+"_PCA2"],
fields['is_set'],
vs.table_name,fields['is_set'])
results = databases[db].execute_query(sql)
ret_dict={}
for r in results:
i= r['id']
del r['id']
ret_dict[i]=r
set_info=None
for item in p.data['ml_peak_class']['sets']:
if item["name"]==set:
set_info=item
return ujson.dumps({"complete":True,"data":ret_dict,"fields":fields,"set":set_info})
else:
return ujson.dumps({"complete":False})
except Exception as e:
app.logger.exception("cannot processs job".format(db,project_id))
return ujson.dumps({"success":False,"msg":"Could not create set"})
def process(self):
try:
p = get_project(self.job.inputs["project_id"])
vs = ViewSet(p.db, p.get_viewset_id())
create_thumbnails_from_ucsc(self.job.inputs["session_url"],
vs,
pixels=self.job.inputs["image_width"],
margins=self.job.inputs.get(
"margins", 0),
job=self.job)
p.set_data("has_images", True)
#send email
url = p.get_url(external=True)
user = db.session.query(User).filter_by(id=self.job.user_id).one()
send_email(user,
"Images Created",
"ucsc_image_job_finished",
url=url)
p.set_data("creating_images_job_status", "complete")
self.complete()
except Exception as e:
app.logger.exception("Error in ucsc upload # {}".format(
self.job.id))
p.set_data("creating_images_job_status", "failed")
self.failed(traceback.format_exc())
def create_compound_column(self,
stages=[],
name="",
final_trans="none",
history={}):
vs = ViewSet(self.db, self.get_viewset_id())
data = vs.create_compound_column(name, stages, final_trans)
history["status"] = "complete"
history["tracks"] = []
f = data["columns"][0]["field"]
history["fields"] = [f]
history["id"] = f
history["graphs"] = [data["graphs"][0]["id"]]
self.add_to_history(history)
data["history"] = history
return data
def create_thumbnails(db, vs_id):
data = request.json
max_y = None
if data['primary_track']['scale'] == "fixed":
max_y = data['primary_track']['max_y']
primary_color = data['primary_track']['color']
vs = ViewSet(db, vs_id)
vs.create_thumbnails(25,
50,
max_y=max_y,
primary_track_color=primary_color,
secondary_track=data.get('secondary_track'),
flanking_region=data.get("margin"),
preview=data.get("preview"),
limit=data.get("limit"))
return ujson.dumps({"success": True})
def add_tagging_column(self):
vs = ViewSet(self.db, self.get_viewset_id())
l_to_f = vs.add_columns([{
"label": "Tags",
"datatype": "text"
}], "current_tags")
field = l_to_f["Tags"]
if not self.data.get("graph_config"):
self.data["graph_config"] = []
self.data["graph_config"].append({
"type": "row_chart",
"title": "Tags",
"param": field,
"id": "current_tags"
})
self.update()
def delete(self, hard=False):
if not hard:
return super().delete()
vs_id = self.data.get("viewset_id")
if vs_id:
vs = ViewSet(self.db, vs_id)
if vs.id != -1:
vs.delete(True)
info = self.data.get("zegami_upload_job")
if info:
delete_job(info["job_id"])
info = self.data.get("ucsc_images")
if info:
delete_job(info["job_id"])
super().delete(True)
def get_view_set_full(db, vs_id):
vs_id = int(vs_id)
vs = ViewSet(db, vs_id)
if request.args.get("simple"):
data = request.json
filters = None
if data:
filters = data.get("filters")
view_data = vs.get_data_simple(filters=filters)
else:
view_data = vs.get_data_for_table()
if request.args.get("project_id"):
pid = int(request.args.get("project_id"))
p = get_project(pid)
perm = p.get_permissions(current_user)
p.data["permission"] = perm
view_data['project_data'] = p.data
return json.dumps(view_data)
def get_cluster_data(self, job_id=0):
from app.jobs.jobs import get_job
j = get_job(job_id)
name = j.get_input_parameter("name")
methods = j.get_input_parameter("methods")
vs = ViewSet(self.db, self.get_viewset_id())
data = vs.get_cluster_data(name, methods)
graph_ids = []
graphs = []
field_graph = name + "_fields"
for method in methods:
graph_ids.append(name + "_" + method)
for graph in self.data["graph_config"]:
if graph['id'] in graph_ids:
graphs.append(graph)
if graph["id"] == field_graph:
graphs.append(graph)
data["graphs"] = graphs
return data
def update_tags(self, tags={}, tag_color_scheme={}):
vs = ViewSet(self.db, self.get_viewset_id())
ctf = vs.data["field_information"]["current_tags"]["Tags"]
update_list = []
for id in tags:
update_list.append({"id": id, ctf: tags[id]})
sql = "UPDATE {} SET {}=NULL".format(vs.table_name, ctf)
databases[self.db].execute_update(sql)
databases[self.db].update_table_with_dicts(update_list, vs.table_name)
self.set_data("tag_color_scheme", tag_color_scheme)
def add_intersections(self, ids, ec):
vs = ViewSet(self.db, self.get_viewset_id())
if ec:
data = vs.add_annotation_fields(ids[0], ec)
else:
data = vs.add_annotations_intersect(ids)
self.refresh_data()
self.data["graph_config"] += data["graphs"]
self.data["browser_config"]["state"] += data["tracks"]
self.update()
tracks = []
graphs = []
for t in data["tracks"]:
tracks.append(t["track_id"])
for g in data["graphs"]:
graphs.append(g["id"])
return {"graphs": graphs, "tracks": tracks, "fields": data["fields"]}
def add_bw_stats(self, url, name):
#copy wigfile
file_name = os.path.split(url)[1]
folder = os.path.join(app.config['TRACKS_FOLDER'], "projects",
str(self.id))
if not os.path.exists(folder):
os.makedirs(folder)
local_file = os.path.join(folder, file_name)
url = shlex.quote(url)
command = "curl {} -o {}".format(url, local_file)
os.system(command)
self.refresh_data()
tid = random_string(5)
self.data["browser_config"]["state"].append({
"url":
local_file.replace("/data", ""),
"track_id":
tid,
"discrete":
True,
"height":
100,
"color":
"#808080",
"scale":
"dynamic",
"short_label":
name,
"type":
"bigwig",
"allow_user_remove":
True,
"format":
"wig"
})
self.update()
vs = ViewSet(self.db, self.get_viewset_id())
fields = vs.add_wig_stats(local_file, name)
return {"fields": fields, "track": tid}
def create_thumbnail_preview(db):
data = request.json
vs = ViewSet(db, data["viewset_id"])
folder = get_temporary_folder()
tn_det = data['thumbnail_details']
vs.data['thumbnail_details'] = tn_det
vs.data["margin"] = tn_det['margin']
vs.data["annotation_sets"] = data["annotations"]
tn = ThumbnailSet(db,
height=int(tn_det['height']),
width=int(tn_det['width']))
ids = list(range(1, 21))
gene_set = 0
if data.get("genes") and vs.db != "other":
gene_set = 1
tn.draw_view_set(vs, specific_views=ids, folder=folder, gene_set=gene_set)
images = []
url_folder = folder.replace("mlv", "")
for i_id in ids:
images.append("{}/tn{}.png".format(url_folder, i_id))
return ujson.dumps({"images": images, "success": True})
def process(self, pword):
try:
p = get_project(self.job.inputs["project_id"])
p.set_data("zegami_upload_job", {
"job_id": self.job.id,
"job_status": "running"
})
vs = ViewSet(p.db, p.get_viewset_id())
credentials = {
"project": self.job.inputs["project"],
"username": self.job.inputs["username"],
"password": pword
}
url = vs.create_zegami_collection(name=p.name,
job=self.job,
credentials=credentials)
p.set_data("zegami_url", url)
vs.upload_images_to_zegami(job=self.job, credentials=credentials)
#send email
user = db.session.query(User).filter_by(id=self.job.user_id).one()
send_email(user,
"Zegami Collection Created",
"zegami_collection_created",
url=url)
self.complete()
except Exception as e:
app.logger.exception("Error in zegami upload # {}".format(
self.job.id))
p.set_data("zegami_upload_job", {
"job_id": self.job.id,
"job_status": "failed"
})
self.failed(traceback.format_exc())
def create_from_project(self, project_id=None, ids=[], fields=[]):
p = get_project(project_id)
vs = ViewSet(p.db, p.get_viewset_id())
new_fields = []
for count, field in enumerate(fields, start=1):
info = vs.fields.get(field)
if not info:
continue
else:
new_fields.append({
"datatype": info["datatype"],
"position": count,
"field": "f{}".format(count),
"name": info["label"]
})
bed = vs.create_basic_bed_file(without_ids=True,
selected_ids=ids,
fields=fields)
files = {"upload_file": bed}
print(bed)
self.create_from_file(files, fields=new_fields, process_file=False)
def delete_columns(self, columns=[]):
vs = ViewSet(self.db, self.get_viewset_id())
vs.remove_columns(columns)
self.refresh_data()
new_graphs = []
for g in self.data["graph_config"]:
param = g.get("param")
if param:
contains_field = False
if isinstance(param, list):
for p in param:
if p in columns:
contains_field = True
break
else:
if param in columns:
contains_field = True
if contains_field:
continue
new_graphs.append(g)
if len(self.data["graph_config"]) != len(new_graphs):
self.data["graph_config"] = new_graphs
self.update()
history = self.data.get("history")
if history:
new_history = []
for h in history:
if h["id"] in columns:
continue
new_history.append(h)
self.set_data("history", new_history)
return self.data["history"]
def label_history(self, history):
self.set_input_parameter("history_id", history["id"])
history["label"] = "Dimension Reduction " + self.job.inputs["name"]
history["job_id"] = self.job.id
info = ""
p = get_project(self.job.inputs["project_id"])
vs = ViewSet(p.db, p.get_viewset_id())
names = []
for f in self.job.inputs["fields"]:
names.append(vs.fields[f]["label"])
info += "Fields: " + ", ".join(names)
info += " \nCluster Methods:" + ", ".join(self.job.inputs["methods"])
info += " \nDimensions: " + str(self.job.inputs["dimensions"])
history["info"] = info
def get_fields(self):
vsid = self.get_viewset_id()
if vsid:
ret_list = []
vs = ViewSet(self.db, vsid)
for field in vs.fields:
item = vs.fields[field]
item["name"] = item["label"]
del item["label"]
item["field"] = field
ret_list.append(item)
return ret_list
else:
return []
def remove_annotation_intersections(self, ids):
vs = ViewSet(self.db, self.get_viewset_id())
info = vs.data["annotation_information"]
for aid in ids:
anno = info.get(str(aid))
if not anno:
continue
del vs.data["field_information"]["Annotations"][anno["label"]]
del vs.data["annotation_information"][aid]
vs.remove_columns([anno['field']])
vs.update()
def process(self):
try:
p = get_project(self.job.inputs["project_id"])
vs = ViewSet(p.db, p.get_viewset_id())
create_thumbnails_from_mlv(
self.job.inputs["tracks"],
vs,
image_width=self.job.inputs["image_width"],
margins=self.job.inputs.get("margins", 0),
job=self.job)
p.refresh_data()
p.set_data("has_images", True)
hid = self.get_input_parameter("history_id")
history = p.get_history(hid)
history["tracks"] = []
history["fields"] = []
history["graphs"] = []
history["status"] = "complete"
p.update()
#send email
url = p.get_url(external=True)
user = db.session.query(User).filter_by(id=self.job.user_id).one()
send_email(user,
"Images Created",
"ucsc_image_job_finished",
url=url)
self.complete()
except Exception as e:
p = get_project(self.job.inputs["project_id"])
hid = self.get_input_parameter("history_id")
history = p.get_history(hid)
history["status"] = "failed"
p.update()
app.logger.exception("Error in creating mlv images # {}".format(
self.job.id))
self.failed(traceback.format_exc())
def delete_tss_distances(self):
vs = ViewSet(self.db, self.get_viewset_id())
fields = vs.data["field_information"].get("TSS").values()
vs.remove_columns(fields)
del vs.data["field_information"]["TSS"]
vs.update()
new_charts = []
for chart in self.data["graph_config"]:
if chart['id'].startswith("_tss"):
continue
new_charts.append(chart)
self.data["graph_config"] = new_charts
del self.data["find_tss_distances_job_id"]
del self.data["find_tss_distances_job_status"]
self.update()
def get_viewset(self, filters=None, offset=None, limit=None):
vs = ViewSet(self.db, self.get_viewset_id())
if (offset or offset == 0) and limit:
off_lim = [int(offset), int(limit)]
if offset == 0:
return {
"views":
vs.get_all_views(filters=filters, offset=off_lim),
"field_information":
vs.data["field_information"],
"fields":
vs.fields,
"sprite_sheets":
vs.data.get("sprite_sheets"),
"annotation_information":
vs.data.get("annotation_information"),
"base_image_url":
"/data/{}/view_sets/{}/thumbnails/tn".format(
self.db, vs.id),
"total":
vs.get_view_number()["count"]
}
else:
return vs.get_all_views(filters=filters, offset=off_lim)
else:
return {
"views":
vs.get_all_views(filters=filters),
"field_information":
vs.data["field_information"],
"fields":
vs.fields,
"sprite_sheets":
vs.data.get("sprite_sheets"),
"annotation_information":
vs.data.get("annotation_information"),
"base_image_url":
"/data/{}/view_sets/{}/thumbnails/tn".format(self.db, vs.id)
}
