def create_analysis_tasks(analysis_id):
print('analysis_id', analysis_id)
# samples = []
# print('hello!')
#create analysis tasks
# analysis = Analysis.objects.()
analysis = Analysis.objects.get(pk=analysis_id)
print(dir(analysis))
params = analysis.params
files = params['files']
for file in files:
task = Task(user=analysis.user)
task.manifest = {}
task.manifest['files'] = [file]
task.manifest['analysis_types'] = params['analysis_types']
task.status = 'new'
task.analysis = analysis
task.action = 'analysis'
task.save()
analysis.tasks.add(task)
# if 'sample_groups' in params:
# samples = Sample.objects.filter(samplegroup_members__in=params['sample_groups'])
# # sample = Sample.objects.first()
# # print(dir(sample))
# for sample in samples:
# print(sample)
# for file in sample.files.all():
# bam_size = 9223372036854775807
# if file.extension == 'bam':
# if file.size < bam_size:
# bamfile = file
# bam_size = file.size
# print('small bam', bamfile.size)
# get smallest bam file
def run_task(request):
if request.method == 'GET':
print(request.GET)
if 'action' in request.GET:
action = request.GET['action']
file_id = request.GET['file_id']
file = File.get_object_or_404(pk=file_id)
if action == "check":
task_manifest = {}
task_manifest['file'] = file.id
task_manifest['action'] = action
task = Task(user=request.user)
task.manifest = task_manifest
task.status = 'new'
task.action = action
task.user = request.user
task.save()
check_file.delay(task.id)
file.status = 'scheduled'
file.save()
return redirect('files-index')
def run_task(request):
if request.method == 'GET':
print(request.GET)
if 'action' in request.GET:
action = request.GET['action']
file_id = request.GET['file_id']
file = File.get_object_or_404(pk=file_id)
if action == "check":
task_manifest = {}
task_manifest['file'] = file.id
task_manifest['action'] = action
task = Task(user=request.user)
task.manifest = task_manifest
task.status = 'new'
task.action = action
task.user = request.user
task.save()
check_file.delay(task.id)
file.status = 'scheduled'
file.save()
return redirect('files-index')
def bulk_action(request):
if request.method == 'POST':
files = request.POST.getlist('files')
action = request.POST['action']
for file_id in files:
# file = File.objects.get(pk=file_id)
if request.user.is_staff:
file = get_object_or_404(File, pk=file_id)
else:
file = get_object_or_404(File, pk=file_id, user=request.user)
if action == "delete":
file.delete()
if action == "check":
task_manifest = {}
task_manifest['file'] = file.id
task_manifest['action'] = action
task = Task(user=request.user)
task.manifest = task_manifest
task.status = 'new'
task.action = action
task.user = request.user
task.save()
check_file.delay(task.id)
file.status = 'scheduled'
file.save()
return redirect('files-index')
def bulk_action(request):
if request.method == 'POST':
files = request.POST.getlist('files')
action = request.POST['action']
for file_id in files:
# file = File.objects.get(pk=file_id)
if request.user.is_staff:
file = get_object_or_404(File, pk=file_id)
else:
file = get_object_or_404(File, pk=file_id, user=request.user)
if action == "delete":
file.delete()
if action == "check":
task_manifest = {}
task_manifest['file'] = file.id
task_manifest['action'] = action
task = Task(user=request.user)
task.manifest = task_manifest
task.status = 'new'
task.action = action
task.user = request.user
task.save()
check_file.delay(task.id)
file.status = 'scheduled'
file.save()
return redirect('files-index')
def bulk_action(request, project_id):
if request.user.is_staff:
project = get_object_or_404(Project, pk=project_id)
else:
project = get_object_or_404(Project, pk=project_id, user=request.user)
if request.method == 'POST':
action = request.POST['action']
print('POST', request.POST)
model = request.POST['model']
if model == 'files':
files = request.POST.getlist('files')
if action == 'analysis':
request.session['files'] = files
request.session['project_id'] = project_id
return redirect('analysis-create')
for file_id in files:
file = File.objects.get(pk=file_id)
if action == "delete":
file.delete()
if action == "qc":
if file.file_type in ['fq', 'fastq', 'bam', 'vcf']:
task = Task()
task.name = 'QC ProjectFile %s' % (file.id)
task.type = 'qc'
task.action = 'qc'
task.status = 'new'
task.manifest = {
'input':file.location,
'project':project.id,
'file':file.id,
'type':'qc',
}
task.save()
run_qc.delay(task.id)
if action == "check":
task_manifest = {}
task_manifest['file'] = file.id
task_manifest['action'] = action
task = Task(user=request.user)
task.manifest = task_manifest
task.status = 'new'
task.action = action
task.user = request.user
task.save()
check_file.delay(task.id)
file.status = 'scheduled'
file.save()
if model == 'samples':
samples = request.POST.getlist('samples')
for sample_id in samples:
sample = ProjectSample.objects.get(pk=sample_id)
if action == "delete":
sample.delete()
return redirect('projects-view', project_id=project_id)
def bulk_action(request, project_id):
if request.user.is_staff:
project = get_object_or_404(Project, pk=project_id)
else:
project = get_object_or_404(Project, pk=project_id, user=request.user)
if request.method == 'POST':
action = request.POST['action']
# print('POST', request.POST)
model = request.POST['model']
if model == 'files':
files = request.POST.getlist('files')
if action == 'analysis':
request.session['files'] = files
request.session['project_id'] = project_id
return redirect('analysis-create')
for file_id in files:
file = File.objects.get(pk=file_id)
if action == "delete":
file.delete()
if action == "qc":
if file.file_type in ['fq', 'fastq', 'bam', 'vcf']:
task = Task()
task.name = 'QC ProjectFile %s' % (file.id)
task.type = 'qc'
task.action = 'qc'
task.status = 'new'
task.manifest = {
'input':file.location,
'project':project.id,
'file':file.id,
'type':'qc',
}
task.save()
run_qc.delay(task.id)
if action == "check":
task_manifest = {}
task_manifest['file'] = file.id
task_manifest['action'] = action
task = Task(user=request.user)
task.manifest = task_manifest
task.status = 'new'
task.action = action
task.user = request.user
task.save()
check_file.delay(task.id)
file.status = 'scheduled'
file.save()
if model == 'samples':
samples = request.POST.getlist('samples')
for sample_id in samples:
sample = ProjectSample.objects.get(pk=sample_id)
if action == "delete":
sample.delete()
return redirect('projects-view', project_id=project_id)
def index(request):
query = ''
args = []
if request.method == 'POST':
print(request.POST)
files = request.POST.getlist('files')
action = request.POST['action']
query = request.POST['query']
# print('query', query)
if action == 'analysis':
request.session['files'] = files
return redirect('analysis-create')
for file_id in files:
# file = File.objects.get(pk=file_id)
if request.user.is_staff:
file = get_object_or_404(File, pk=file_id)
else:
file = get_object_or_404(File, pk=file_id, user=request.user)
if action == "delete":
file.delete()
else:
task_manifest = {}
task_manifest['file'] = file.id
task_manifest['action'] = action
task = Task(user=request.user)
task.manifest = task_manifest
task.status = 'new'
task.action = action
task.user = request.user
task.save()
if action == "check":
check_file.delay(task.id)
if action == "compress":
compress_file.delay(task.id)
file.status = 'scheduled'
file.save()
status = request.POST.getlist('status')
print('status', status)
if len(status[0]) > 0:
args.append(Q(status__in=status))
extension = request.POST.getlist('extension')
if len(extension[0]) > 0:
print('extension', extension)
args.append(Q(extension__in=extension))
print('args', args)
# Or the Q object with the ones remaining in the list
if request.method == 'GET':
print(request.GET)
if 'orderby' in request.GET:
orderby = request.GET['orderby'][0]
order = request.GET['order'][0]
if order == 'desc':
order_string = '-{}'.format(orderby)
else:
order_string = orderby
else:
order_string = 'name'
if request.user.is_staff: #status='scheduled' size=0
files = File.objects.filter(location__icontains=query,
*args).order_by('sample') #size
else:
files = File.objects.filter(user=request.user).order_by(order_string)
files = File.objects.all().order_by('-id')
files_summary = {}
files_summary['status'] = dict(
Counter(files.values_list('status', flat=True)))
files_summary['file_type'] = dict(
Counter(files.values_list('file_type', flat=True)))
files_summary['extension'] = dict(
Counter(files.values_list('extension', flat=True)))
# files_summary['total_size'] = sum(files.values_list('size', flat=True))
files_summary = 0
context = {
'query': query,
'files': files,
'n_files': len(files),
'files_summary': files_summary
}
return render(request, 'files/index.html', context)
def upload(request):
if request.method == 'POST':
form = UploadForm(request.POST, request.FILES)
if form.is_valid():
if request.user.is_authenticated:
file = File.objects.create(user=request.user, status='new')
else:
file = File.objects.create(user=None, status='new')
file.local_file = request.FILES.get('file')
print('file')
print(request.FILES.get('file'))
filename = file.local_file.name.split('.')
new_filename = []
for tag in filename:
new_filename.append(slugify(tag))
file.local_file.name = ".".join(new_filename)
# print('file.location ', file.location)
#get name from inside vcf file
#clean this...
file.name = str(os.path.splitext(file.local_file.name)[0]).replace(
'.vcf', '').replace('.gz', '').replace('.rar', '').replace(
'.zip', '').replace('._', ' ').replace('.', ' ')
# file.shared_with_groups = form.cleaned_data['shared_with_groups']
# file.shared_with_groups.set(form.cleaned_data['shared_with_groups'])
file.save()
f = file.local_file
#fix permissions
#os.chmod("%s/genomes/%s/" % (settings.BASE_DIR, file.user), 0777)
if request.user.is_authenticated:
file_path = "%s/media/%s/%s" % (settings.BASE_DIR,
slugify(file.user), file.id)
else:
file_path = "%s/media/public/%s" % (settings.BASE_DIR, file.id)
os.chmod(file_path, 0o777)
file.location = '/' + file.local_file.url
# AnnotateVariants.delay(file.id)
task_manifest = {}
task_manifest['file'] = file.id
task_manifest['action'] = 'check'
task = Task(user=request.user)
task.manifest = task_manifest
task.status = 'new'
task.action = 'check'
task.save()
check_file.delay(task.id)
file.status = 'scheduled'
file.save()
data = {
'files': [{
'deleteType': 'DELETE',
'name': file.name,
'url': '',
'thumbnailUrl': '',
'type': 'image/png',
'deleteUrl': '',
'size': f.size
}]
}
response = JSONResponse(data, mimetype=response_mimetype(request))
response['Content-Disposition'] = 'inline; filename=files.json'
return response
else:
print(form.errors)
else:
form = UploadForm()
return render(request, 'files/upload.html', {'form': form})
def index(request):
query = ''
args = []
if request.method == 'POST':
# form = FileForm(request.POST)
# if form.is_valid():
# pass
# print(request.POST)
files = request.POST.getlist('files')
action = request.POST['action']
query = request.POST['query']
order_string ='sample'
# print('query', query)
if action == 'analysis':
request.session['files'] = files
return redirect('analysis-create')
for file_id in files:
# file = File.objects.get(pk=file_id)
if request.user.is_staff:
file = get_object_or_404(File, pk=file_id)
else:
file = get_object_or_404(File, pk=file_id, user=request.user)
if action == "delete":
file.delete()
else:
task_manifest = {}
task_manifest['file'] = file.id
task_manifest['action'] = action
task = Task(user=request.user)
task.manifest = task_manifest
task.status = 'new'
task.action = action
task.user = request.user
task.save()
if action == "check":
check_file.delay(task.id)
if action == "compress":
compress_file.delay(task.id)
file.status = 'scheduled'
file.save()
status = request.POST.getlist('status')
print('status', status)
if len(status[0]) > 0:
args.append(Q(status__in=status))
extension = request.POST.getlist('extension')
if len(extension[0]) > 0:
print('extension',extension)
args.append(Q(extension__in=extension))
print('args', args)
else:
# form = FileForm()
print(request.GET)
if 'orderby' in request.GET:
orderby = request.GET['orderby']
order = request.GET['order'][0]
if order == 'desc':
order_string = '-{}'.format(orderby)
else:
order_string = orderby
else:
order_string = 'name'
# print(order_string, 'order_string')
if request.user.is_staff:#status='scheduled' size=0
# files = File.objects.filter(location__icontains=query, *args).order_by(order_string)#size
files = File.objects.filter(location__regex=query, *args).order_by(order_string)#size
else:
files = File.objects.filter(user=request.user).order_by(order_string)
#sample__isnull=True
# files = File.objects.filter(*args).order_by('-id')
files_summary = {}
files_summary['status'] = dict(Counter(files.values_list('status', flat=True)))
files_summary['file_type'] = dict(Counter(files.values_list('file_type', flat=True)))
files_summary['extension'] = dict(Counter(files.values_list('extension', flat=True)))
# files_summary['total_size'] = sum(files.values_list('size', flat=True))
# files_summary = 0
filetypes = []
n_files = []
for filetype in files_summary['extension']:
# print(filetype, files_summary['extension'][filetype])
filetypes.append(filetype)
n_files.append(files_summary['extension'][filetype])
files_summary['filetypes'] = filetypes
files_summary['n_files'] = n_files
context = {
# 'form': form,
'query':query,
'files':files,
'n_files':len(files),
'files_summary':files_summary
}
return render(request, 'files/index.html', context)
def upload(request):
if request.method == 'POST':
form = UploadForm(request.POST, request.FILES)
if form.is_valid():
if request.user.is_authenticated:
file = File.objects.create(user=request.user, status='new')
else:
file = File.objects.create(user=None, status='new')
file.local_file = request.FILES.get('file')
print('file')
print(request.FILES.get('file'))
filename = file.local_file.name.split('.')
new_filename = []
for tag in filename:
new_filename.append(slugify(tag))
file.local_file.name = ".".join(new_filename)
# print('file.location ', file.location)
#get name from inside vcf file
#clean this...
file.name= str(os.path.splitext(file.local_file.name)[0]).replace('.vcf','').replace('.gz','').replace('.rar','').replace('.zip','').replace('._',' ').replace('.',' ')
# file.shared_with_groups = form.cleaned_data['shared_with_groups']
# file.shared_with_groups.set(form.cleaned_data['shared_with_groups'])
file.save()
f = file.local_file
#fix permissions
#os.chmod("%s/genomes/%s/" % (settings.BASE_DIR, file.user), 0777)
if request.user.is_authenticated:
file_path = "%s/media/%s/%s" % (settings.BASE_DIR, slugify(file.user), file.id)
else:
file_path = "%s/media/public/%s" % (settings.BASE_DIR, file.id)
os.chmod(file_path, 0o777)
file.location = '/'+file.local_file.url
# AnnotateVariants.delay(file.id)
task_manifest = {}
task_manifest['file'] = file.id
task_manifest['action'] = 'check'
task = Task(user=request.user)
task.manifest = task_manifest
task.status = 'new'
task.action = 'check'
task.save()
check_file.delay(task.id)
file.status = 'scheduled'
file.save()
data = {'files': [{'deleteType': 'DELETE', 'name': file.name, 'url': '', 'thumbnailUrl': '', 'type': 'image/png', 'deleteUrl': '', 'size': f.size}]}
response = JSONResponse(data, mimetype=response_mimetype(request))
response['Content-Disposition'] = 'inline; filename=files.json'
return response
else:
print(form.errors)
else:
form = UploadForm()
return render(request, 'files/upload.html', {'form':form})
