def fetch_learning(request):
import json
query_string = request.GET['hash'] + ',' + request.GET['type'] + ',' + str(get_config('env', 'cpu')) \
+ ',' + str(get_config('env', 'memory')) + ',' + str(os_to_int())
api_bus = get_config('program', 'api',
1) + '/Index/share/q/' + query_string
try:
res_data = urlopen(
Request(api_bus, headers={"User-Agent":
"BioQueue-client"})).read()
res = json.loads(res_data.read())
session_dict = {
'hash': request.GET['hash'],
'type': request.GET['type'],
'a': res['a'],
'b': res['b'],
'r': res['r'],
}
request.session['learning'] = session_dict
template = loader.get_template('ui/fetch_learning.html')
context = {
'step': res,
}
return success(template.render(context))
except Exception as e:
return error(api_bus)
def delete_upload_file(request, f):
file_path = os.path.join(
get_config('env', 'workspace'),
str(request.user.queuedb_profile_related.delegate.id),
base64.b64decode(f).decode())
delete_file(file_path)
fm_path = os.path.join(get_config('env', 'workspace'), "file_comment", f)
if os.path.exists(fm_path):
delete_file(fm_path)
return success('Deleted')
def show_workspace_files(user_id, special_type='uploads', block_files=None):
import time
user_files = []
if special_type != "uploads" and special_type != "refs":
user_path = special_type
else:
user_path = os.path.join(get_config('env', 'workspace'), str(user_id),
special_type)
if not os.path.exists(user_path):
os.makedirs(user_path)
for root, dirs, files in os.walk(user_path):
for file_name in files:
file_full_path = os.path.join(root, file_name)
subs = ""
if root != user_path:
subs = root.replace(user_path + "/", "")
tmp = dict()
tmp['name'] = os.path.join(subs, file_name)
tmp['file_size'] = os.path.getsize(file_full_path)
tmp['file_create'] = time.ctime(os.path.getctime(file_full_path))
tmp['trace'] = base64.b64encode(
os.path.join(special_type, tmp['name']).encode()).decode()
tmp['raw'] = base64.b64encode(tmp['name'].encode()).decode()
if block_files is not None and tmp['raw'] not in block_files:
user_files.append(tmp)
user_files = sorted(user_files, key=lambda user_files: user_files['name'])
return user_files
def show_upload_files(request, special_type='uploads'):
user_path = os.path.join(
get_config('env', 'workspace'),
str(request.user.queuedb_profile_related.delegate.id), special_type)
user_files = []
if not os.path.exists(user_path):
try:
os.makedirs(user_path)
except Exception as e:
return render(request, 'ui/error.html', {'error_msg': e})
for root, dirs, files in os.walk(user_path):
for file_name in files:
warn_msg = []
file_full_path = os.path.join(root, file_name)
file_name_warning = 0
if file_full_path.find(" ") != -1:
file_name_warning = 1
warn_msg.append("White space found in the file path")
else:
try:
file_full_path.encode("ascii")
except UnicodeEncodeError:
file_name_warning = 1
warn_msg.append("Non-ASCII code found in the file path")
file_path = file_full_path.replace(user_path + '\\', '') \
.replace(user_path + '/', '').replace(user_path, '')
user_files.append(
(file_path, file_name_warning, ";".join(warn_msg)))
context = {'user_files': sorted(user_files, key=lambda x: x[0])}
return render(request, 'ui/show_uploads.html', context)
def handle_uploaded_file(f):
import os
file_name = os.path.join(get_config('env', 'batch_job'),
rand_sig() + '.txt')
with open(file_name, 'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
return file_name
def delete_job_file(request, f):
file_path = os.path.join(
get_config('env', 'workspace'),
str(request.user.queuedb_profile_related.delegate.id),
base64.b64decode(f).decode())
delete_file(file_path)
return success('Deleted')
def check_disk_quota_lock(user):
disk_limit = get_config('env', 'disk_quota')
if disk_limit:
if get_user_folder_size(user) < int(disk_limit):
return 1
else:
return 0
else:
return 1
def get_disk_quota_info(user):
try:
disk_pool = int(get_config('env', 'disk_quota'))
disk_used = get_user_folder_size(user)
disk_perc = int(round(disk_used / disk_pool * 100))
except Exception as e:
print(e)
disk_pool = disk_used = disk_perc = 0
return disk_pool, disk_used, disk_perc
def create_protocol(request):
import hashlib
if request.method == 'POST':
protocol_form = CreateProtocolForm(request.POST)
if protocol_form.is_valid():
try:
cd = protocol_form.cleaned_data
if ProtocolList.objects.filter(name=cd['name'], user=request.user.queuedb_profile_related.delegate).exists():
return error('Duplicate record!')
protocol = ProtocolList(name=cd['name'], description=cd['description'],
user=request.user.queuedb_profile_related.delegate)
protocol.save()
softwares = request.POST.getlist('software', '')
parameters = request.POST.getlist('parameter', '')
version_checks = request.POST.getlist('version_check', '')
virtual_environments = request.POST.getlist('env', '')
steps = []
try:
protocol_id_trace = ProtocolList.objects.get(id=protocol.id)
except Exception as e:
return error(e)
for index, software in enumerate(softwares):
if parameters[index]:
m = hashlib.md5((software + ' ' + parameters[index].strip()).encode())
env = virtual_environments[index]
if env != "":
try:
env = VirtualEnvironment.objects.get(id=int(env))
except:
env = None
else:
env = None
steps.append(Step(software=software,
parameter=parameters[index],
version_check=version_checks[index],
env=env,
parent=protocol_id_trace,
hash=m.hexdigest(),
step_order=index + 1,
user=request.user.queuedb_profile_related.delegate))
Step.objects.bulk_create(steps)
archive_protocol(request, protocol_id_trace.id)
return success('Your protocol have been created!')
except Exception as e:
return error(e)
else:
return error(str(protocol_form.errors))
else:
if request.user.is_superuser:
available_env = VirtualEnvironment.objects.all()
else:
available_env = VirtualEnvironment.objects.filter(user=request.user.queuedb_profile_related.delegate).all()
context = {'api_bus': get_config('program', 'api', 1),
'user_envs': available_env, }
return render(request, 'ui/add_protocol.html', context)
def delete_job_file_tree(request, f):
try:
if f is not None and f != "":
user_dir = os.path.join(
get_config('env', 'workspace'),
str(request.user.queuedb_profile_related.delegate.id))
job_path = os.path.join(user_dir, f)
import shutil
if os.path.exists(job_path) and not os.path.samefile(
user_dir, job_path):
shutil.rmtree(job_path, ignore_errors=True)
except Exception as e:
print(e)
def query_usage(request):
"""
This function should only be called when the user is using IE8 or IE9
:param request:
:return:
"""
api_bus = get_config(
'program', 'api',
1) + '/Kb/findSoftwareUsage?software=' + request.POST['software']
try:
req = Request(api_bus, headers={"User-Agent": "BioQueue-client"})
res = urlopen(req).read()
return HttpResponse(res.decode("utf-8"))
except Exception as e:
return error(api_bus)
def show_job_log(request):
if request.method == 'POST':
query_job_form = QueryJobLogForm(request.POST)
if query_job_form.is_valid():
cd = query_job_form.cleaned_data
suffix = ".log" if cd["std_out"] else ".err"
log_path = os.path.join(get_config('env', 'log'),
str(cd['job']) + suffix)
try:
from worker.bases import get_job_log
return success(get_job_log(log_path))
except Exception as e:
return error(e)
else:
return error(str(query_job_form.errors))
else:
return error('Method error')
def send_file_as_reference(request, f):
user_workspace = os.path.join(
get_config('env', 'workspace'),
str(request.user.queuedb_profile_related.delegate.id))
file_path = os.path.join(user_workspace,
base64.b64decode(f.replace('f/', '')).decode())
ref_folder = os.path.join(user_workspace, 'refs')
if not os.path.exists(file_path) or not os.path.isfile(file_path):
return error('Cannot find the file.')
if not os.path.exists(ref_folder) or not os.path.isdir(ref_folder):
try:
os.makedirs(ref_folder)
except Exception as e:
return error(e)
import shutil
shutil.move(file_path, ref_folder)
return success("")
def create_reference_shortcut(request):
reference_form = CreateReferenceForm(request.POST)
if reference_form.is_valid():
cd = reference_form.cleaned_data
if cd['source'] == 'upload' or cd['source'] == 'job':
file_path = os.path.join(
get_config('env', 'workspace'),
str(request.user.queuedb_profile_related.delegate.id),
base64.b64decode(cd['path']))
ref = Reference(
name=cd['name'],
path=file_path,
description=cd['description'],
user=request.user.queuedb_profile_related.delegate,
)
ref.save()
return success(ref.id)
else:
return error(str(reference_form.errors))
def download_file(request, job_id, f):
try:
if job_id != 0:
job = Job.objects.get(id=job_id)
file_path = os.path.join(
job.run_dir,
str(request.user.queuedb_profile_related.delegate.id),
base64.b64decode(f.replace('f/', '').encode()).decode())
else:
file_path = os.path.join(
get_config("env", "workspace"),
str(request.user.queuedb_profile_related.id),
base64.b64decode(f.replace('f/', '').encode()).decode())
if os.path.exists(file_path):
return download(file_path)
else:
return render(request, "ui/error.html",
{"error_msg": "Cannot locate the file"})
except Exception as e:
return render(request, "ui/error.html", {"error_msg": str(e)})
def clean_dead_folder(request):
from shutil import rmtree
jobs_containers = os.path.join(get_config('env', 'workspace'), str(request.user.queuedb_profile_related.delegate.id))
job_results = set(
[re["result"] for re in Job.objects.filter(user_id=str(request.user.queuedb_profile_related.delegate.id)).values('result')])
protected_folders = set(["refs", "bin", "uploads", "archives", "OVERRIDE_UPLOAD"])
death_counter = 0
failed = 0
if os.path.exists(jobs_containers):
for job_result in os.listdir(jobs_containers):
if job_result not in job_results and job_result not in protected_folders:
death_counter += 1
try:
abs_path = os.path.join(jobs_containers, job_result)
if os.path.isdir(abs_path):
rmtree(abs_path)
except:
failed += 1
return success("%d/%d dead folder detected. %d of them failed to be cleaned." % (death_counter,
len(job_results),
failed))
def import_protocol_by_fetch(request):
if request.method == 'POST':
form = FetchRemoteProtocolForm(request.POST)
if form.is_valid():
try:
api_bus = get_config('program', 'api', 1) + '/Protocol/exportProtocolStdout?sig=' + request.POST['uid']
try:
req = Request(api_bus, headers={"User-Agent": "BioQueue-client"})
protocol_raw = urlopen(req).read()
import json
import hashlib
protocol_json = json.loads(protocol_raw.decode("utf-8"))
if ProtocolList.objects.filter(name=protocol_json['name'],
user=request.user.queuedb_profile_related.delegate).exists():
return error('Duplicate record!')
protocol = ProtocolList(name=protocol_json['name'], user=request.user.queuedb_profile_related.delegate)
protocol.save()
steps = []
predictions = []
try:
protocol_id_trace = ProtocolList.objects.get(id=protocol.id)
except Exception as e:
return error(e)
for step in protocol_json['step']:
m = hashlib.md5((step['software'] + ' ' + step['parameter'].strip()).encode())
# m.update(step['software'] + ' ' + step['parameter'].strip())
steps.append(Step(software=step['software'],
parameter=step['parameter'],
parent=protocol_id_trace,
hash=m.hexdigest(),
step_order=step['step_order'],
user=request.user.queuedb_profile_related.delegate))
if 'cpu_a' in step.keys() and 'cpu_b' in step.keys() and 'cpu_r' in step.keys():
if 'cpu' in step.keys():
if step['cpu'] != get_config('env', 'cpu'):
continue
if Prediction.objects.filter(step_hash=m.hexdigest(), type=3).exists():
continue
else:
predictions.append(Prediction(a=step['cpu_a'],
b=step['cpu_b'],
r=step['cpu_r'],
type=3,
step_hash=m.hexdigest()))
if 'mem_a' in step.keys() and 'mem_b' in step.keys() and 'mem_r' in step.keys():
if 'mem' in step.keys():
if step['mem'] != get_config('env', 'mem'):
continue
if Prediction.objects.filter(step_hash=m.hexdigest(), type=2).exists():
continue
else:
predictions.append(Prediction(a=step['mem_a'],
b=step['mem_b'],
r=step['mem_r'],
type=2,
step_hash=m.hexdigest()))
if 'vrt_a' in step.keys() and 'vrt_b' in step.keys() and 'vrt_r' in step.keys():
if 'mem' in step.keys():
if step['mem'] != get_config('env', 'mem'):
continue
if Prediction.objects.filter(step_hash=m.hexdigest(), type=2).exists():
continue
else:
predictions.append(Prediction(a=step['vrt_a'],
b=step['vrt_b'],
r=step['vrt_r'],
type=4,
step_hash=m.hexdigest()))
if 'disk_a' in step.keys() and 'disk_b' in step.keys() and 'disk_r' in step.keys():
if Prediction.objects.filter(step_hash=m.hexdigest(), type=1).exists():
continue
else:
predictions.append(Prediction(a=step['disk_a'],
b=step['disk_b'],
r=step['disk_r'],
type=1,
step_hash=m.hexdigest()))
Step.objects.bulk_create(steps)
if len(predictions):
Prediction.objects.bulk_create(predictions)
ref_list = list()
for key, value in enumerate(protocol_json['reference']):
try:
_ = Reference.objects.get(name=value['name'], user=request.user.queuedb_profile_related.delegate)
ref_list.append({'name': value['name'],
'description': value['description'],
'status': 1, })
except:
ref_list.append({'name': value['name'],
'description': value['description'],
'status': 0, })
from django.template import RequestContext, loader
template = loader.get_template('ui/import_protocol.html')
return success(template.render({'ref_list': ref_list}))
except Exception as e:
return error(e)
except Exception as e:
return error(e)
else:
return error(form.errors)
else:
return error('Error method')
def build_plain_protocol(request, protocol_id, stand_alone=True):
"""
Convert protocol into a plain format
:param request:
:param protocol_id: int, protocol id
:return: int or string, 1 means no such a protocol, 2 means no permission to access the protocol
"""
protocol_data = dict()
try:
protocol_parent = ProtocolList.objects.get(id=protocol_id)
except ProtocolList.DoesNotExist:
# protocol doesn't exist
return '', 1
if protocol_parent.check_owner(request.user.queuedb_profile_related.delegate):
steps = Step.objects.filter(parent=protocol_parent.id)
else:
return '', 2
protocol_data['name'] = protocol_parent.name
protocol_data['step'] = []
for step in steps:
tmp = {
'software': step.software,
'parameter': step.parameter,
'hash': step.hash,
'step_order': step.step_order,
}
protocol_data['step'].append(tmp)
if stand_alone:
references = {}
reference_list = Reference.objects.filter(user=request.user.queuedb_profile_related.delegate).all()
protocol_ref = {}
# references.extend([reference.name for reference in reference_list])
for reference in reference_list:
references[reference.name] = reference.description
wildcard_pattern = re.compile("\\{\\{(.*?)\\}\\}", re.IGNORECASE | re.DOTALL)
for i, step in enumerate(steps):
try:
for wildcard in re.findall(wildcard_pattern, step.parameter):
wildcard = wildcard.split(':')[0]
if wildcard in references.keys() and wildcard not in protocol_ref.keys():
protocol_ref[wildcard] = references[wildcard]
equations = Prediction.objects.filter(step_hash=step.hash)
cpu_a = cpu_b = cpu_r = mem_a = mem_b = mem_r = disk_a = disk_b = disk_r = vrt_a = vrt_b = vrt_r = 0
for equation in equations:
if equation.type == 1:
disk_a = equation.a
disk_b = equation.b
disk_r = equation.r
elif equation.type == 2:
mem_a = equation.a
mem_b = equation.b
mem_r = equation.r
elif equation.type == 3:
cpu_a = equation.a
cpu_b = equation.b
cpu_r = equation.r
elif equation.type == 4:
vrt_a = equation.a
vrt_b = equation.b
vrt_r = equation.r
protocol_data['step'][i]['cpu'] = get_config('env', 'cpu')
protocol_data['step'][i]['memory'] = get_config('env', 'memory')
protocol_data['step'][i]['os'] = os_to_int()
protocol_data['step'][i]['cpu_a'] = cpu_a
protocol_data['step'][i]['cpu_b'] = cpu_b
protocol_data['step'][i]['cpu_r'] = cpu_r
protocol_data['step'][i]['mem_a'] = mem_a
protocol_data['step'][i]['mem_b'] = mem_b
protocol_data['step'][i]['mem_r'] = mem_r
protocol_data['step'][i]['vrt_a'] = vrt_a
protocol_data['step'][i]['vrt_b'] = vrt_b
protocol_data['step'][i]['vrt_r'] = vrt_r
protocol_data['step'][i]['disk_a'] = disk_a
protocol_data['step'][i]['disk_b'] = disk_b
protocol_data['step'][i]['disk_r'] = disk_r
except:
pass
# protocol_data['step'].append(tmp)
protocol_data['reference'] = protocol_ref
return protocol_data['name'], build_json_protocol(protocol_data)
def file_support(request):
if request.method == "GET":
fs_form = FileSupportForm(request.GET)
if fs_form.is_valid():
cd = fs_form.cleaned_data
real_file = base64.b64decode(cd["file"]).decode()
_, ext = os.path.splitext(real_file)
from ..tools import get_maintenance_protocols
protocol_name = "%s (%s, %s)" % (cd["support"], cd["exp"], ext)
try:
protocol_record = ProtocolList.objects.get(
name=protocol_name,
user_id=request.user.queuedb_profile_related.delegate.id)
except ProtocolList.DoesNotExist:
# build protocol
if request.user.queuedb_profile_related.delegate.id == 0:
return error("Log in first.")
protocol_parent = ProtocolList(
name=protocol_name,
user_id=request.user.queuedb_profile_related.delegate.id)
protocol_parent.save()
steps = list()
maintenance_protocols = get_maintenance_protocols()
step_order = 1
if cd["support"] == "gg":
if ext != ".gz":
model = __import__("ui.maintenance_protocols.compress",
fromlist=["gzip"])
else:
model = __import__(
"ui.maintenance_protocols.decompress",
fromlist=["gunzip"])
else:
if cd["support"] in maintenance_protocols:
model = __import__("ui.maintenance_protocols." +
cd["support"],
fromlist=[cd["support"]])
else:
return error("No support found.")
step_order, sub_steps = model.get_sub_protocol(
Step, protocol_parent, step_order)
for sub_step in sub_steps:
steps.append(sub_step)
try:
Step.objects.bulk_create(steps)
protocol_record = protocol_parent
except:
protocol_parent.delete()
return error('Fail to save the protocol.')
job = Job(
protocol_id=protocol_record.id,
parameter=';',
run_dir=get_config('env', 'workspace'),
user=request.user.queuedb_profile_related.delegate,
input_file="{{Uploaded:%s}}" % real_file,
job_name="%s-%s" % (cd['support'], real_file),
)
try:
job.save()
return success('Push the task into job queue.')
except:
return error('Fail to save the job.')
else:
return error(str(fs_form.errors))
def import_protocol(request):
if request.method == 'POST':
form = BatchJobForm(request.POST, request.FILES)
if form.is_valid():
file_name = handle_uploaded_file(request.FILES['job_list'])
try:
with open(file_name) as f:
protocol_raw = f.read()
import json
protocol_json = json.loads(protocol_raw)
if ProtocolList.objects.filter(name=protocol_json['name'],
user=request.user.queuedb_profile_related.delegate).exists():
return error('Duplicate record!')
protocol = ProtocolList(name=protocol_json['name'], user=request.user.queuedb_profile_related.delegate)
protocol.save()
steps = []
predictions = []
try:
protocol_id_trace = ProtocolList.objects.get(id=protocol.id)
except Exception as e:
return error(e)
for step in protocol_json['step']:
m = hashlib.md5((step['software'] + ' ' + step['parameter'].strip()).encode())
# m.update(step['software'] + ' ' + step['parameter'].strip())
steps.append(Step(software=step['software'],
parameter=step['parameter'],
parent=protocol_id_trace,
hash=m.hexdigest(),
step_order=step['step_order'],
user=request.user.queuedb_profile_related.delegate))
if 'cpu_a' in step.keys() and 'cpu_b' in step.keys() and 'cpu_r' in step.keys():
if Prediction.objects.filter(step_hash=m.hexdigest(), type=3).exists():
continue
else:
predictions.append(Prediction(a=step['cpu_a'],
b=step['cpu_b'],
r=step['cpu_r'],
type=3,
step_hash=m.hexdigest()))
if 'mem_a' in step.keys() and 'mem_b' in step.keys() and 'mem_r' in step.keys():
if Prediction.objects.filter(step_hash=m.hexdigest(), type=2).exists():
continue
else:
predictions.append(Prediction(a=step['mem_a'],
b=step['mem_b'],
r=step['mem_r'],
type=2,
step_hash=m.hexdigest()))
if 'vrt_a' in step.keys() and 'vrt_b' in step.keys() and 'vrt_r' in step.keys():
if 'mem' in step.keys():
if step['mem'] != get_config('env', 'mem'):
continue
if Prediction.objects.filter(step_hash=m.hexdigest(), type=2).exists():
continue
else:
predictions.append(Prediction(a=step['vrt_a'],
b=step['vrt_b'],
r=step['vrt_r'],
type=4,
step_hash=m.hexdigest()))
if 'disk_a' in step.keys() and 'disk_b' in step.keys() and 'disk_r' in step.keys():
if Prediction.objects.filter(step_hash=m.hexdigest(), type=1).exists():
continue
else:
predictions.append(Prediction(a=step['disk_a'],
b=step['disk_b'],
r=step['disk_r'],
type=1,
step_hash=m.hexdigest()))
Step.objects.bulk_create(steps)
archive_protocol(request, protocol.id)
if len(predictions):
Prediction.objects.bulk_create(predictions)
return HttpResponseRedirect('/ui/query-protocol')
except Exception as e:
return render(request, 'ui/error.html', {'error_msg': e})
else:
return render(request, 'ui/error.html', {'error_msg': str(form.errors)})
else:
return render(request, 'ui/error.html', {'error_msg': 'Error method'})
def batch_job(request):
if request.method == 'POST':
form = BatchJobForm(request.POST, request.FILES)
if form.is_valid():
file_name = handle_uploaded_file(request.FILES['job_list'])
try:
protocol_cache = dict()
try:
ws = Workspace.objects.get(id=request.session['workspace'])
except Workspace.DoesNotExist:
ws = None
with open(file_name) as f:
jobs = f.readlines()
job_list = []
for job in jobs:
configurations = job.split('\n')[0].split('\t')
if len(configurations) == 4:
if check_disk_quota_lock(
request.user.queuedb_profile_related.
delegate.id):
protocol_id = int(configurations[0])
if protocol_id not in protocol_cache:
try:
protocol = ProtocolList.objects.get(
id=protocol_id)
protocol_cache[protocol_id] = (int(
protocol.user_id), protocol.ver)
except Exception as e:
return render(request, 'ui/error.html',
{'error_msg': e})
if protocol_cache[
protocol_id][0] == request.user.queuedb_profile_related.delegate.id or request.user.is_staff or \
protocol_cache[protocol_id][0] == 0:
job_list.append(
Job(protocol_id=protocol_id,
protocol_ver=protocol_cache[
protocol_id][1],
job_name=configurations[1],
input_file=configurations[2],
parameter=configurations[3],
run_dir=get_config(
'env', 'workspace'),
user=request.user.
queuedb_profile_related.delegate,
workspace=ws))
else:
return render(
request, 'ui/error.html', {
'error_msg':
'You are not the owner of the protocol(%s)'
% protocol_id
})
else:
return render(
request, 'ui/error.html', {
'error_msg':
'You have exceed the disk quota limit! Please delete some files!'
})
else:
return render(
request, 'ui/error.html', {
'error_msg':
'Your job list file must contain three columns.'
})
Job.objects.bulk_create(job_list)
return HttpResponseRedirect('/ui/query-job')
except Exception as e:
return render(request, 'ui/error.html', {'error_msg': e})
else:
return render(request, 'ui/error.html',
{'error_msg': str(form.errors)})
def add_job(request):
if request.method == 'POST':
job_form = SingleJobForm(request.POST)
if job_form.is_valid():
cd = job_form.cleaned_data
try:
if cd['parameter'].find(';') == -1:
cd['parameter'] += ';'
protocol = ProtocolList.objects.get(id=cd['protocol'])
if protocol.check_owner(
request.user.queuedb_profile_related.delegate):
try:
if 'workspace' in request.session:
ws = Workspace.objects.get(
id=request.session['workspace'])
else:
ws = None
except Workspace.DoesNotExist:
ws = None
job = Job(
protocol_id=cd['protocol'],
protocol_ver=protocol.ver,
job_name=cd['job_name'],
parameter=cd['parameter'],
run_dir=get_config('env', 'workspace'),
user=request.user.queuedb_profile_related.delegate,
input_file=cd['input_files'],
workspace=ws,
)
if check_disk_quota_lock(
request.user.queuedb_profile_related.delegate.id):
job.save()
Audition(operation="Created a new job",
related_job=job,
job_name=job.job_name,
prev_par=job.parameter,
new_par=job.parameter,
prev_input=job.input_file,
current_input=job.input_file,
protocol=job.protocol.name,
protocol_ver=job.protocol_ver,
resume_point=job.resume,
user=job.user).save()
return success('Successfully added job into queue.')
else:
return error(
'You have exceed the disk quota limit! Please delete some files!'
)
else:
return error('You are not owner of the protocol.')
except Exception as e:
return error(e)
return error(str(job_form.errors))
else:
if request.user.is_staff:
available_protocol = ProtocolList.objects.all()
else:
available_protocol = ProtocolList.objects.filter(
Q(user=request.user.queuedb_profile_related.delegate)
| Q(user=None)).all()
dt, du, dp = get_disk_quota_info_with_cache(
request.user.queuedb_profile_related.delegate.id)
return render(
request, 'ui/add_job.html', {
'form': SingleJobForm,
'user_protocols': available_protocol,
't_disk': dt,
'u_disk': du,
'disk_per': dp
})
def install_reference(request):
if request.method == "POST":
import hashlib
from json import loads
from ..tools import get_maintenance_protocols
ref_info = loads(request.POST["tool"])
if type(ref_info["software"]) == str or type(
ref_info["software"]) == unicode:
protocol_name = ref_info["how_get"] + "_" + ref_info[
"compression"] + "_" + ref_info["software"]
else:
protocol_name = ref_info["how_get"] + "_" + ref_info[
"compression"] + "_" + "_".join(ref_info["software"])
try:
protocol_parent = ProtocolList.objects.get(name=protocol_name,
user_id=0)
except ProtocolList.DoesNotExist:
protocol_parent = ProtocolList(name=protocol_name, user_id=0)
protocol_parent.save()
steps = list()
maintenance_protocols = get_maintenance_protocols()
step_order = 1
# download
if ref_info["how_get"] not in maintenance_protocols and ref_info[
"how_get"] != "n":
protocol_parent.delete()
return error("No protocol to fetch the data")
else:
model = __import__("ui.maintenance_protocols." +
ref_info["how_get"],
fromlist=[ref_info["how_get"]])
step_order, sub_steps = model.get_sub_protocol(
Step, protocol_parent, step_order)
for sub_step in sub_steps:
steps.append(sub_step)
# decompress
if ref_info[
"compression"] not in maintenance_protocols and ref_info[
"compression"] != "n":
protocol_parent.delete()
return error("No protocol to decompress (%s) the data" %
ref_info["compression"])
else:
if ref_info["compression"] != "n":
model = __import__("ui.maintenance_protocols." +
ref_info["compression"],
fromlist=[ref_info["compression"]])
step_order, sub_steps = model.get_sub_protocol(
Step, protocol_parent, step_order)
for sub_step in sub_steps:
steps.append(sub_step)
# post-decompression
if type(ref_info['software']) == str or type(
ref_info['software']) == unicode:
step_order += 1
m = hashlib.md5((ref_info['software'] + ' ' +
ref_info['parameter'].strip()).encode())
# m.update(ref_info['software'] + ' ' + ref_info['parameter'].strip())
steps.append(
Step(software=ref_info['software'],
parameter=ref_info['parameter'].strip(),
parent=protocol_parent,
hash=m.hexdigest(),
step_order=step_order,
user_id=0))
elif len(ref_info['software']) == len(
ref_info['parameter']) and len(ref_info['software']) >= 1:
for ind, software in enumerate(ref_info['software']):
step_order += 1
m = hashlib.md5(
(software + ' ' +
ref_info['parameter'][ind].strip()).encode())
# m.update(software + ' ' + ref_info['parameter'][ind].strip())
steps.append(
Step(software=software,
parameter=ref_info['parameter'][ind].strip(),
parent=protocol_parent,
hash=m.hexdigest(),
step_order=step_order,
user_id=0))
# move to user's ref folder
steps.append(
Step(software="mv",
parameter="{{CompileTargets}} {{UserRef}}",
parent=protocol_parent,
user_id=0,
hash='d885188751fc204ad7bbdf63fc6564df',
step_order=step_order + 1))
Step.objects.bulk_create(steps)
user_ref_dir = os.path.join(
os.path.join(
get_config('env', 'workspace'),
str(request.user.queuedb_profile_related.delegate.id)), 'ref')
if not os.path.exists(user_ref_dir):
try:
os.makedirs(user_ref_dir)
except:
return error('Fail to create your reference folder')
if "target_files" in ref_info.keys():
mv_parameter = " ".join(ref_info["target_files"].split(";"))
else:
mv_parameter = "{{LastOutput}}"
job = Job(
protocol_id=protocol_parent.id,
parameter='UserRef=%s;CompileTargets=%s' %
(user_ref_dir, mv_parameter),
run_dir=get_config('env', 'workspace'),
user_id=request.user.queuedb_profile_related.delegate.id,
input_file=ref_info["url"],
)
target_files = ref_info["target_files"].split(";")
# create links to references
ref_list = list()
for target_file in target_files:
ref_list.append(
Reference(
name=ref_info['name'],
path=os.path.join(user_ref_dir, target_file),
description=ref_info['description'],
user_id=request.user.queuedb_profile_related.delegate.id,
))
if len(ref_list) > 0:
Reference.objects.bulk_create(ref_list)
try:
job.save()
return success('Push the task into job queue.')
except:
return error('Fail to save the job.')
else:
api_bus = get_config('program', 'ref_repo_search_api', 1)
tool_addr = get_config('program', 'ref_repo', 1)
return render(request, 'ui/install_ref.html', {
'ab': api_bus,
'ta': tool_addr
})
def fetch_data(request):
if request.method == "POST":
from ..tools import get_maintenance_protocols
protocol_name = "Fetch Data From EBI ENA"
try:
protocol_record = ProtocolList.objects.get(name=protocol_name,
user_id=0)
except ProtocolList.DoesNotExist:
# build protocol
protocol_parent = ProtocolList(name=protocol_name, user_id=0)
protocol_parent.save()
steps = list()
maintenance_protocols = get_maintenance_protocols()
# download
step_order = 1
if "download" not in maintenance_protocols:
protocol_parent.delete()
return error("No protocol to fetch the data")
else:
model = __import__("ui.maintenance_protocols.download",
fromlist=["download"])
step_order, sub_steps = model.get_sub_protocol(
Step, protocol_parent, step_order)
for sub_step in sub_steps:
steps.append(sub_step)
# decompress
if "gunzip" not in maintenance_protocols:
protocol_parent.delete()
return error("No protocol to decompress (gz) the data")
else:
model = __import__("ui.maintenance_protocols.gunzip",
fromlist=["gunzip"])
step_order, sub_steps = model.get_sub_protocol(
Step, protocol_parent, step_order)
for sub_step in sub_steps:
steps.append(sub_step)
# move to user's uploads folder
steps.append(
Step(software="mv",
parameter="{{LastOutput}} {{UserUploads}}",
parent=protocol_parent,
user_id=0,
hash='c5f8bf22aff4c9fd06eb0844e6823d5f',
step_order=step_order))
try:
Step.objects.bulk_create(steps)
protocol_record = protocol_parent
except:
protocol_parent.delete()
return error('Fail to save the protocol.')
user_upload_dir = os.path.join(
os.path.join(
get_config('env', 'workspace'),
str(request.user.queuedb_profile_related.delegate.id)),
'uploads')
if not os.path.exists(user_upload_dir):
try:
os.makedirs(user_upload_dir)
except:
return error('Fail to create your uploads folder')
from ..ena import query_download_link_from_ebi
links = query_download_link_from_ebi(request.POST["acc"])
if len(links) > 0:
job = Job(
job_name=request.POST["acc"],
protocol_id=protocol_record.id,
parameter='UserUploads=%s;' % user_upload_dir,
run_dir=get_config('env', 'workspace'),
user=request.user.queuedb_profile_related.delegate,
input_file=";".join(links),
)
try:
job.save()
return success("<br>".join(links))
except:
return error('Fail to save the job.')
else:
return error("No result found.")
else:
return render(request, 'ui/fetch_data.html')
def settings(request):
if request.method == 'POST':
set_config('env', 'workspace', request.POST['path'])
set_config('env', 'cpu', request.POST['cpu'])
set_config('env', 'cpu_m', request.POST['cpu_m'])
set_config('env', 'memory', request.POST['mem'])
set_config('env', 'disk_quota', request.POST['dquota'])
set_config('ml', 'confidence_weight_disk', request.POST['dcw'])
set_config('ml', 'confidence_weight_mem', request.POST['mcw'])
set_config('ml', 'confidence_weight_cpu', request.POST['ccw'])
set_config('ml', 'threshold', request.POST['ccthr'])
if request.POST['mailhost'] != '':
set_config('mail', 'notify', 'on')
set_config('mail', 'mail_host', request.POST['mailhost'])
set_config('mail', 'mail_port', request.POST['mailport'])
set_config('mail', 'mail_user', request.POST['mailuser'])
set_config('mail', 'mail_password', request.POST['mailpassword'])
if request.POST['protocol'] == 'ssl':
set_config('mail', 'ssl', 'true')
elif request.POST['protocol'] == 'tls':
set_config('mail', 'tls', 'true')
else:
set_config('mail', 'notify', 'off')
if request.POST['cluster_type'] != '':
set_config('cluster', 'type', request.POST['cluster_type'])
set_config('cluster', 'cpu', request.POST['job_cpu'])
set_config('cluster', 'queue', request.POST['job_dest'])
set_config('cluster', 'mem', request.POST['job_mem'])
set_config('cluster', 'vrt', request.POST['job_vrt'])
set_config('cluster', 'walltime', request.POST['job_wt'])
else:
set_config('cluster', 'type', '')
set_config('cluster', 'cpu', '')
set_config('cluster', 'queue', '')
set_config('cluster', 'mem', '')
set_config('cluster', 'vrt', '')
set_config('cluster', 'walltime', '')
return HttpResponseRedirect('/ui/settings')
else:
from worker.cluster_support import get_cluster_models
try:
if get_config('mail', 'ssl') == 'true':
mail_protocol = 'ssl'
elif get_config('mail', 'tls') == 'true':
mail_protocol = 'tls'
else:
mail_protocol = 'nm'
configuration = {
'run_folder':
get_config('env', 'workspace'),
'cpu':
get_config('env', 'cpu'),
'cpu_m':
get_config('env', 'cpu_m'),
'memory':
get_config('env', 'memory'),
'disk_quota':
get_config('env', 'disk_quota'),
'threshold':
get_config('ml', 'threshold'),
'disk_confidence_weight':
get_config('ml', 'confidence_weight_disk'),
'mem_confidence_weight':
get_config('ml', 'confidence_weight_mem'),
'cpu_confidence_weight':
get_config('ml', 'confidence_weight_cpu'),
'max_disk':
round((get_disk_free(get_config('env', 'workspace')) +
get_disk_used(get_config('env', 'workspace'))) /
1073741824),
'free_disk':
round(
get_disk_free(get_config('env', 'workspace')) /
1073741824),
'mail_host':
get_config('mail', 'mail_host'),
'mail_port':
get_config('mail', 'mail_port'),
'mail_user':
get_config('mail', 'mail_user'),
'mail_password':
get_config('mail', 'mail_password'),
'mail_protocol':
mail_protocol,
'cluster_models':
get_cluster_models(),
'cluster_type':
get_config('cluster', 'type'),
'job_cpu':
get_config('cluster', 'cpu'),
'job_dest':
get_config('cluster', 'queue'),
'job_mem':
get_config('cluster', 'mem'),
'job_vrt':
get_config('cluster', 'vrt'),
'job_wt':
get_config('cluster', 'walltime'),
'rv':
get_config('program', 'latest_version', 1),
'cv':
get_bioqueue_version(),
}
except Exception as e:
return render(request, 'ui/error.html', {'error_msg': e})
return render(request, 'ui/settings.html', configuration)
def save_job(archive_id, logger):
def file_md5(file_path):
hash_md5 = hashlib.md5()
with open(file_path, "rb") as fh:
for chunk in iter(lambda: fh.read(4096), b""):
hash_md5.update(chunk)
return hash_md5.hexdigest()
try:
archive = FileArchive.objects.get(id=archive_id)
logger.info("Working on %s" % archive)
try:
prev_same_archive = FileArchive.objects.filter(
protocol=archive.protocol,
protocol_ver=archive.protocol_ver,
files=archive.files).exclude(file_md5s="ph")
logger.info("# md5 caches" % len(prev_same_archive))
except:
prev_same_archive = None
logger.info("No md5 cache")
md5_flag = 0
tared_flag = 0
uploaded_flag = 0
files = json.loads(archive.files.replace("'", "\""))
if prev_same_archive is not None:
for psa in prev_same_archive:
md5_flag = 1
archive.file_md5s = psa.file_md5s
if psa.archive_file != "ph":
archive.archive_file = psa.archive_file
tared_flag = 1
if psa.file_id_remote != "ph":
archive.file_id_remote = psa.file_id_remote
uploaded_flag = 1
archive.save()
if not md5_flag:
logger.info("Calculating MD5 sums")
md5_sums = []
for file in files:
md5_sums.append(file_md5(file))
archive.file_md5s = ",".join(md5_sums)
archive.save()
if not tared_flag:
logger.info("Compressing files")
import tarfile
user_archives = archive.user.queuedb_profile_related.delegate.queuedb_profile_related.archive_folder
if user_archives != "":
if not os.path.exists(user_archives):
try:
os.makedirs(user_archives)
except Exception as e:
logger.exception(e)
archive_folder = ""
if user_archives == "":
user_archives = os.path.join(
get_config("env", "workspace"),
str(archive.user.queuedb_profile_related.delegate.id),
"archives")
if not os.path.exists(user_archives):
os.makedirs(user_archives)
tan = "%s.tar.gz" % os.path.join(user_archives, str(archive.id))
tf = tarfile.open(tan, mode="w:gz")
for f in files:
tf.add(f, arcname=os.path.basename(f))
tf.close()
archive.archive_file = tan
archive.save()
# if not uploaded_flag:
# logger.info("Uploading files")
# try:
# g_info = GoogleDriveConnection.objects.get(user=archive.created_by)
# g = GoogleDrive(token_file=g_info.credential_pickle)
# folder_tag, folder_id = g.already_folder("BioQueue_Share")
# if not folder_tag:
# folder_id = g.create_folder("BioQueue_Share")
# file_id = g.upload_file(file_name="%d.tar.gz" % archive.id, file_path=archive.archive_file,
# folder_id=folder_id, description=archive.description)
# archive.file_id_remote = file_id
# archive.status = 1
# archive.save()
#
# users = archive.shared_with.split(",")
# g.share_with_person(file_id, users, msg=archive.description+"\n Protocol ver: "+archive.protocol_ver)
# except GoogleDriveConnection.DoesNotExist:
# archive.comment = "No Google Drive connection available for account %s" % archive.created_by.username
# archive.save()
# except Exception as e:
# archive.comment(e)
# logger.error(e)
# archive.save()
except FileArchive.DoesNotExist as e:
logger.error(e)
return False
def register_sample(request):
if request.method == 'POST':
reg_form = AddSampleForm(request.POST)
if reg_form.is_valid():
cd = reg_form.cleaned_data
try:
exp = Experiment.objects.get(id=cd['experiment'])
try:
Sample.objects.get(
user=request.user.queuedb_profile_related.delegate,
file_path=cd["file_path"])
return error("Sample already exist!")
except Sample.DoesNotExist:
# if user prefers not to use the default upload folder
# then BioQueue will create symbolic links in the uploads
# folder for user
if request.user.queuedb_profile_related.upload_folder != "":
first_order_soft_link = os.path.join(
get_config('env', 'workspace'),
str(request.user.queuedb_profile_related.delegate.
id), "OVERRIDE_UPLOAD")
if not os.path.exists(
first_order_soft_link) or not os.path.islink(
first_order_soft_link):
os.symlink(
request.user.queuedb_profile_related.
upload_folder, first_order_soft_link)
expected_upload_path = os.path.join(
get_config('env', 'workspace'),
str(request.user.queuedb_profile_related.delegate.
id), "uploads")
for file in cd["inner_path"].split(";"):
real_path = base64.b64decode(file).decode("utf-8")
dst_path = os.path.join(
expected_upload_path,
os.path.split(real_path)[1])
if not os.path.exists(dst_path):
os.symlink(src=os.path.join(
"../OVERRIDE_UPLOAD", real_path),
dst=dst_path)
else:
return error(
"This sample is already registered")
Sample(name=cd["name"],
file_path=";".join([
base64.b64decode(fp.encode()).decode()
for fp in cd["file_path"].split(";") if fp != ""
]),
user=request.user.queuedb_profile_related.delegate,
experiment=exp,
attribute=cd["attribute"],
inner_path=cd["inner_path"]).save()
return success("Sample registered")
except Sample.MultipleObjectsReturned:
return error("Duplicate records!")
except Exception as e:
return error(e)
else:
return error(str(reg_form.errors))
else:
block_files = Sample.objects.filter(
user=request.user.queuedb_profile_related.delegate)
block_file_real = set()
for bf in block_files:
for rbf in bf.inner_path.split(";"):
block_file_real.add(rbf)
if request.user.queuedb_profile_related.upload_folder != "":
ufs = show_workspace_files(
request.user.queuedb_profile_related.delegate.id,
request.user.queuedb_profile_related.upload_folder,
block_files=block_file_real)
else:
ufs = show_workspace_files(
request.user.queuedb_profile_related.delegate.id,
'uploads',
block_files=block_file_real)
context = {
'user_files':
ufs,
'user_ref_files':
show_workspace_files(
request.user.queuedb_profile_related.delegate.id, 'refs'),
'experiments':
Experiment.objects.filter(),
}
return render(request, 'ui/register_sample.html', context)
