def process(self):
try:
#do they contain tabix index bed files
for pid in self.job.inputs["ids"]:
p = get_project(pid)
if not p.data.get("bed_file"):
p.create_anno_bed_file()
p = get_project(self.job.inputs["project_id"])
data = p.add_intersections(self.job.inputs["ids"],
self.job.inputs.get("extra_columns"))
hid = self.get_input_parameter("history_id")
p.refresh_data()
history = p.get_history(hid)
if history:
history["tracks"] = data["tracks"]
history["fields"] = data["fields"]
history["graphs"] = data["graphs"]
history["status"] = "complete"
p.update()
self.complete()
except Exception as e:
app.logger.exception(
"Cannot process Annotation IntersectionJob # {}".format(
self.job.id))
p.refresh_data()
hid = self.get_input_parameter("history_id")
history = p.get_history(hid)
history["status"] = "failed"
p.update()
self.failed(traceback.format_exc())
def process(self):
try:
p = get_project(self.job.inputs["project_id"])
vs = ViewSet(p.db, p.get_viewset_id())
data = vs.add_ts_starts(overlap_column=True,
go_levels=self.job.inputs.get(
"go_levels", 0))
p.refresh_data()
p.data["graph_config"] += data["graphs"]
hid = self.get_input_parameter("history_id")
history = p.get_history(hid)
history["tracks"] = []
history["fields"] = data["fields"]
history["graphs"] = []
for g in data["graphs"]:
history["graphs"].append(g["id"])
history["status"] = "complete"
p.update()
self.complete()
except Exception as e:
app.logger.exception(
"Cannot process AnnotationIntersectionJob # {}".format(
self.job.id))
p.refresh_data()
hid = self.get_input_parameter("history_id")
history = p.get_history(hid)
history["status"] = "failed"
p.update()
self.failed(traceback.format_exc())
def process(self):
p = get_project(self.job.inputs["project_id"])
try:
name = self.job.inputs["name"]
dimensions = self.job.inputs.get("dimensions", 2)
data = p.cluster_by_fields(self.job.inputs["fields"], name,
self.job.inputs["methods"], dimensions)
p.refresh_data()
hid = self.get_input_parameter("history_id")
history = p.get_history(hid)
history["tracks"] = []
history["fields"] = data["fields"]
history["graphs"] = data["graphs"]
history["status"] = "complete"
p.update()
self.complete()
except Exception as e:
app.logger.exception(
"Cannot process ClusterByFieldsJob # {}".format(self.job.id))
p.refresh_data()
hid = self.get_input_parameter("history_id")
history = p.get_history(hid)
history["status"] = "failed"
p.update()
self.failed(traceback.format_exc())
def execute_project_action(project_id):
try:
p = get_project(project_id)
data=request.json
if data:
arguments= data.get("args",{})
method= data.get("method")
#special case when files are inolved
if len(request.files)>0:
data = ujson.loads(request.form.get("data"))
method=data.get("method")
meth_info = p.methods.get(method)
if meth_info and p.has_edit_permission(current_user):
#need edit permission to upload file
arguments=data.get("arguments",{})
arguments["files"]={}
for fid in request.files:
f = request.files[fid]
filepath = save_file(f)
arguments["files"][fid]=filepath
else:
raise Exception("No permission to upload file")
return ujson.dumps(p.execute_action(method,current_user,arguments))
except Exception as e:
return ujson.dumps({"msg":"Cannot execute action","success":False})
def process(self):
p = get_project(self.job.inputs["project_id"])
try:
tracks = []
fields = []
for n, wl in enumerate(self.job.inputs["wig_locations"]):
name = self.job.inputs["wig_names"][n]
self.set_status("Processing " + name)
info = p.add_bw_stats(wl, name)
tracks.append(info["track"])
fields = fields + info["fields"]
hid = self.get_input_parameter("history_id")
p.refresh_data()
history = p.get_history(hid)
history["tracks"] = tracks
history["fields"] = fields
history["graphs"] = []
history["status"] = "complete"
p.update()
self.complete()
except Exception as e:
app.logger.exception("Cannot process PeakStatsJob # {}".format(
self.job.id))
p.refresh_data()
hid = self.get_input_parameter("history_id")
history = p.get_history(hid)
history["status"] = "failed"
p.update()
self.failed(traceback.format_exc())
def process(self):
try:
p = get_project(self.job.inputs["project_id"])
vs = ViewSet(p.db, p.get_viewset_id())
create_thumbnails_from_ucsc(self.job.inputs["session_url"],
vs,
pixels=self.job.inputs["image_width"],
margins=self.job.inputs.get(
"margins", 0),
job=self.job)
p.set_data("has_images", True)
#send email
url = p.get_url(external=True)
user = db.session.query(User).filter_by(id=self.job.user_id).one()
send_email(user,
"Images Created",
"ucsc_image_job_finished",
url=url)
p.set_data("creating_images_job_status", "complete")
self.complete()
except Exception as e:
app.logger.exception("Error in ucsc upload # {}".format(
self.job.id))
p.set_data("creating_images_job_status", "failed")
self.failed(traceback.format_exc())
def get_project_fields(id):
p = get_project(id)
perm = p.get_permissions(current_user)
if not perm:
return ujson.dumps({"success":False,"msg":"You do not have permission"})
return ujson.dumps(p.get_fields())
def execute_action_async(project_id, method, args):
try:
p = get_project(project_id)
meth = getattr(p, method)
meth(**args)
except Exception as e:
app.logger.exception("Error")
def remove_deleted_projects():
sql = "SELECT id FROM projects WHERE is_deleted=True ORDER BY id "
res = databases['system'].execute_query(sql)
for r in res:
p=get_project(r['id'])
p.delete(True)
return ujson.dumps({"success":True})
def fix_tags_in_set(db,project_id):
set_name = request.form.get("set")
tag_set = request.form.get("tags")
if tag_set:
tag_set=ujson.loads(tag_set)
p = get_project(project_id)
if not p.has_edit_permission(current_user):
return ujson.dumps({"success":False,"msg":"You do not have permission"})
vs =ViewSet(db,p.data['viewset_id'])
set_info=None
for item in p.data['ml_peak_class']['sets']:
if item['name']==set_name:
set_info=item
break
tags =vs.add_tags_to_set(set_name,set_info['label'],
zegami_url=set_info.get('zegami_url'),
tags=tag_set,p=p)
total=0
for t in tags:
total+=tags[t]
set_info['tags_submitted']=True
set_info['tags']=tags
set_info['tags']['all']=set_info['size']
tag_field=vs.data["field_information"]['sets'][set_name][set_name+"_tags"]
p.update()
return ujson.dumps({"success":True,"set":set_info,"tag_field":tag_field})
def check_tag_set_clustering(db,job_id):
try:
job = get_job(job_id)
job.check_status()
if job.job.status=="complete":
si = job.job.inputs['set_info']
project_id=si[0]
set= si[1]
p= get_project(project_id)
vs_id= p.data['initial_peak_calls']['view_set_id']
vs = ViewSet(db,vs_id)
fields= vs.data['field_information']["sets"][set]
sql ="SELECT id,{},{},{},{},{} from {} WHERE {}='True'".format(fields[set+"_tSNE1"],fields[set+"_tSNE2"],
fields[set+"_PCA1"],fields[set+"_PCA2"],
fields['is_set'],
vs.table_name,fields['is_set'])
results = databases[db].execute_query(sql)
ret_dict={}
for r in results:
i= r['id']
del r['id']
ret_dict[i]=r
set_info=None
for item in p.data['ml_peak_class']['sets']:
if item["name"]==set:
set_info=item
return ujson.dumps({"complete":True,"data":ret_dict,"fields":fields,"set":set_info})
else:
return ujson.dumps({"complete":False})
except Exception as e:
app.logger.exception("cannot processs job".format(db,project_id))
return ujson.dumps({"success":False,"msg":"Could not create set"})
def remove_deleted_projects():
sql = "SELECT id FROM projects WHERE is_deleted=True ORDER BY id"
res = databases['system'].execute_query(sql)
for r in res:
p = get_project(r['id'])
print(p.id)
p.delete(True)
def get_track_details(project_id,view_id):
p= get_project(project_id)
vsid = p.get_viewset_id()
sql = "SELECT chromosome AS chr, start, finish AS end FROM view_set_{} WHERE id =%s".format(vsid)
res = databases[p.db].execute_query(sql,(view_id,))
return ujson.dumps({
"tracks":p.data["peak_tracks"],
"position":res[0]
})
def process(self):
try:
p = get_project(self.job.inputs["project_id"])
vs = ViewSet(p.db, p.get_viewset_id())
create_thumbnails_from_mlv(
self.job.inputs["tracks"],
vs,
image_width=self.job.inputs["image_width"],
margins=self.job.inputs.get("margins", 0),
job=self.job)
p.refresh_data()
p.set_data("has_images", True)
hid = self.get_input_parameter("history_id")
history = p.get_history(hid)
history["tracks"] = []
history["fields"] = []
history["graphs"] = []
history["status"] = "complete"
p.update()
#send email
url = p.get_url(external=True)
user = db.session.query(User).filter_by(id=self.job.user_id).one()
send_email(user,
"Images Created",
"ucsc_image_job_finished",
url=url)
self.complete()
except Exception as e:
p = get_project(self.job.inputs["project_id"])
hid = self.get_input_parameter("history_id")
history = p.get_history(hid)
history["status"] = "failed"
p.update()
app.logger.exception("Error in creating mlv images # {}".format(
self.job.id))
self.failed(traceback.format_exc())
def delete_object(id):
success=True
p = get_project(id)
if not p.has_edit_permission(current_user):
return ujson.dumps({"success":False,"msg":"You do not have permission"})
try:
p.delete()
except Exception as e:
app.logger.exception("There was an error deleting project# {}".format(id))
success=False
return ujson.dumps({"success":success})
def create_tag_set(db,project_id):
try:
p= get_project(project_id)
parent= request.form.get("parent")
tag = request.form.get("tag")
label= request.form.get("label")
data =p.create_new_tag_set(tag,parent,label,current_user.id)
return ujson.dumps({"success":True,"data":data})
except Exception as e:
app.logger.exception("cannot create tag set database:{} project:{}".format(db,project_id))
return ujson.dumps({"success":False,"msg":"Could not create set"})
def unshare_object_with(object_id,user_id):
ret_val = {"success":True}
try:
p = get_project(object_id)
if not p.has_edit_permission(current_user):
return ujson.dumps({"success":False,"msg":"You do not have permission"})
p.unshare_with(user_id)
except Exception as e:
app.logger.exception("Could not share object genome:{},object id:{},user_id{}".format(db,object_id,user_id))
ret_val["success"]=False
return ujson.dumps(ret_val)
def update_object(id):
p=get_project(id)
if not p.has_edit_permission(current_user):
return ujson.dumps({"success":False,"msg":"You do not have permission"})
data = request.json
try:
for key in data:
p.set_data(key,data[key])
return ujson.dumps({"success":True})
except Exception as e:
app.logger.exception("There was a problem updating the project {} with data {}".format(id,data))
return ujson.dumps({"success":False,"msg":"There was an error"})
def project_page(type, project_id):
p = get_project(project_id)
if not p.has_view_permission(current_user):
return refuse_permission()
template, args = p.get_template(request.args)
all_args = {
"project_id": project_id,
"project_name": p.name,
"project_type": type,
"description": p.description
}
all_args.update(args)
return render_template(template, **all_args)
def make_object_public(object_id):
ret_val = {"success":True}
try:
p = get_project(object_id)
if not p.has_edit_permission(current_user):
return ujson.dumps({"success":False,"msg":"You do not have permission"})
p.make_public()
ret_val["url"] = p.get_url(external=True)
except Exception as e:
app.logger.exception("Could not make project public - object id:{}".format(object_id))
ret_val["success"]=False
ret_val['message']="There was a problem making the project public"
return ujson.dumps(ret_val)
def label_history(self, history):
self.set_input_parameter("history_id", history["id"])
history["label"] = "Dimension Reduction " + self.job.inputs["name"]
history["job_id"] = self.job.id
info = ""
p = get_project(self.job.inputs["project_id"])
vs = ViewSet(p.db, p.get_viewset_id())
names = []
for f in self.job.inputs["fields"]:
names.append(vs.fields[f]["label"])
info += "Fields: " + ", ".join(names)
info += " \nCluster Methods:" + ", ".join(self.job.inputs["methods"])
info += " \nDimensions: " + str(self.job.inputs["dimensions"])
history["info"] = info
def get_view_set_full(db, vs_id):
vs_id = int(vs_id)
vs = ViewSet(db, vs_id)
if request.args.get("simple"):
data = request.json
filters = None
if data:
filters = data.get("filters")
view_data = vs.get_data_simple(filters=filters)
else:
view_data = vs.get_data_for_table()
if request.args.get("project_id"):
pid = int(request.args.get("project_id"))
p = get_project(pid)
perm = p.get_permissions(current_user)
p.data["permission"] = perm
view_data['project_data'] = p.data
return json.dumps(view_data)
def find_orphan_viewsets():
sql = "SELECT id,genome FROM projects"
res = databases["system"].execute_query(sql)
genomes = {}
for r in res:
p = get_project(r["id"])
vsid = p.get_viewset_id()
if vsid:
li = genomes.get(r["genome"])
if not li:
li = []
genomes[r["genome"]] = li
li.append(vsid)
for genome in genomes:
print(genome)
sql = "SELECT id FROM view_sets WHERE NOT (id = ANY (%s)) ORDER BY id"
res = databases[genome].execute_query(sql, (genomes[genome], ))
for r in res:
print(r['id'])
def check_tag_set_classifying(job_id):
try:
job = get_job(job_id)
job.check_status()
if job.job.status=="complete" or job.job.status=="failed":
project_id= job.job.inputs['project_id']
set=job.job.inputs['set']
set_info=None
p= get_project(project_id)
for item in p.data['ml_peak_class']['sets']:
if item['name']==set:
set_info=item
data=None
if request.form.get("get_fields"):
data = p.get_set_data(set)
return ujson.dumps({"complete":True,"set":set_info,"field_data":data})
else:
return ujson.dumps({"complete":False})
except Exception as e:
app.logger.exception("cannot processs job".format(db,project_id))
return ujson.dumps({"success":False,"msg":"Could not create set"})
def get_project_data(id):
p = get_project(id)
perm = p.get_permissions(current_user)
if not perm:
return ujson.dumps({"success":False,"msg":"You do not have permission"})
gd = app.config["GENOME_DATABASES"][p.db]
genome_info={
"label":gd["label"],
"build":p.db,
"gene_description":gd["gene_description"]
}
return ujson.dumps({
"name":p.name,
"type":p.type,
"genome":p.db,
"description":p.description,
"data":p.get_data(),
"permission":perm,
"status":p.status,
"id":p.id,
"genome_info":genome_info
})
def process(self, pword):
try:
p = get_project(self.job.inputs["project_id"])
p.set_data("zegami_upload_job", {
"job_id": self.job.id,
"job_status": "running"
})
vs = ViewSet(p.db, p.get_viewset_id())
credentials = {
"project": self.job.inputs["project"],
"username": self.job.inputs["username"],
"password": pword
}
url = vs.create_zegami_collection(name=p.name,
job=self.job,
credentials=credentials)
p.set_data("zegami_url", url)
vs.upload_images_to_zegami(job=self.job, credentials=credentials)
#send email
user = db.session.query(User).filter_by(id=self.job.user_id).one()
send_email(user,
"Zegami Collection Created",
"zegami_collection_created",
url=url)
self.complete()
except Exception as e:
app.logger.exception("Error in zegami upload # {}".format(
self.job.id))
p.set_data("zegami_upload_job", {
"job_id": self.job.id,
"job_status": "failed"
})
self.failed(traceback.format_exc())
def create_from_project(self, project_id=None, ids=[], fields=[]):
p = get_project(project_id)
vs = ViewSet(p.db, p.get_viewset_id())
new_fields = []
for count, field in enumerate(fields, start=1):
info = vs.fields.get(field)
if not info:
continue
else:
new_fields.append({
"datatype": info["datatype"],
"position": count,
"field": "f{}".format(count),
"name": info["label"]
})
bed = vs.create_basic_bed_file(without_ids=True,
selected_ids=ids,
fields=fields)
files = {"upload_file": bed}
print(bed)
self.create_from_file(files, fields=new_fields, process_file=False)
def get_share_details_for_object(id):
p = get_project(id)
if not p.has_edit_permission(current_user):
return ujson.dumps({"success":False,"msg":"You do not have permission"})
return ujson.dumps(p.get_shared_with())
def update_permission_type_for_object(id,uid,level):
p = get_project(id)
if not p.has_edit_permission(current_user):
return ujson.dumps({"success":False,"msg":"You do not have permission"})
return ujson.dumps(p.change_permission_type(level,uid))
