def store_analysis_result(request): """Stores the reporting of a transferred analysis result file, checks its md5""" if request.method != 'POST': return HttpResponseNotAllowed(permitted_methods=['POST']) data = request.POST # create analysis file if ('client_id' not in data or data['client_id'] != settings.ANALYSISCLIENT_APIKEY): return HttpResponseForbidden() # FIXME nextflow to file this, then poll rawstatus/md5success # before deleting rundir etc, or report taskfail # Reruns lead to trying to store files multiple times, avoid that: anashare = ServerShare.objects.get(name=settings.ANALYSISSHARENAME) try: ftypeid = {x.name: x.id for x in StoredFileType.objects.all()}[data['ftype']] except KeyError: return HttpResponseForbidden('File type does not exist') try: sfile = StoredFile.objects.get(rawfile_id=data['fn_id'], filetype_id=ftypeid) except StoredFile.DoesNotExist: print('New transfer registered, fn_id {}'.format(data['fn_id'])) sfile = StoredFile(rawfile_id=data['fn_id'], filetype_id=ftypeid, servershare=anashare, path=data['outdir'], filename=data['filename'], md5='', checked=False) sfile.save() AnalysisResultFile.objects.create(analysis_id=data['analysis_id'], sfile=sfile) else: print('Analysis result already registered as transfer, client asks for new ' 'MD5 check after a possible rerun. Running MD5 check.') create_file_job('get_md5', sfile.id) return HttpResponse()
def singlefile_qc(rawfile, storedfile): """This method is only run for detecting new incoming QC files""" add_to_qc(rawfile, storedfile) jobutil.create_file_job('convert_single_mzml', storedfile.id, queue=settings.QUEUE_QCPWIZ) start_qc_analysis(rawfile, storedfile, settings.LONGQC_NXF_WF_ID, settings.LONGQC_FADB_ID)
def start_qc_analysis(rawfile, storedfile, wf_id, dbfn_id): analysis = Analysis(user_id=settings.QC_USER_ID, name='{}_{}_{}'.format(rawfile.producer.name, rawfile.name, rawfile.date)) analysis.save() jobutil.create_file_job('run_longit_qc_workflow', storedfile.id, analysis.id, wf_id, dbfn_id)
def file_transferred(request): """Treats POST requests with: - fn_id Starts checking file MD5 in background """ if request.method == 'POST': try: fn_id = request.POST['fn_id'] client_id = request.POST['client_id'] ftype = request.POST['ftype'] fname = request.POST['filename'] except KeyError as error: print('POST request to file_transferred with missing parameter, ' '{}'.format(error)) return HttpResponseForbidden() try: check_producer(client_id) except Producer.DoesNotExist: return HttpResponseForbidden() tmpshare = ServerShare.objects.get(name=settings.TMPSHARENAME) try: RawFile.objects.get(pk=fn_id) except RawFile.DoesNotExist: print('File has not been registered yet, cannot transfer') return JsonResponse({ 'fn_id': request.POST['fn_id'], 'state': 'error' }) try: file_transferred = StoredFile.objects.get(rawfile_id=fn_id, filetype=ftype) except StoredFile.DoesNotExist: print('New transfer registered, fn_id {}'.format(fn_id)) file_transferred = StoredFile(rawfile_id=fn_id, filetype=ftype, servershare=tmpshare, path='', filename=fname, md5='', checked=False) file_transferred.save() jobutil.create_file_job('get_md5', file_transferred.id) else: print('File already registered as transfer, client asks for new ' 'MD5 check after a possible retransfer. Running MD5 check.') jobutil.create_file_job('get_md5', file_transferred.id) finally: return JsonResponse({ 'fn_id': request.POST['fn_id'], 'state': 'ok' }) else: return HttpResponseNotAllowed(permitted_methods=['POST'])
def file_transferred(request): """Treats POST requests with: - fn_id Starts checking file MD5 in background """ if request.method == 'POST': try: fn_id = request.POST['fn_id'] client_id = request.POST['client_id'] ftype = request.POST['ftype'] fname = request.POST['filename'] except KeyError as error: print('POST request to file_transferred with missing parameter, ' '{}'.format(error)) return HttpResponseForbidden() try: check_producer(client_id) except Producer.DoesNotExist: return HttpResponseForbidden() tmpshare = ServerShare.objects.get(name=settings.TMPSHARENAME) try: RawFile.objects.get(pk=fn_id) except RawFile.DoesNotExist: print('File has not been registered yet, cannot transfer') return JsonResponse({'fn_id': request.POST['fn_id'], 'state': 'error'}) try: ftypeid = {x.name: x.id for x in StoredFileType.objects.all()}[ftype] except KeyError: return HttpResponseForbidden('File type does not exist') try: file_transferred = StoredFile.objects.get(rawfile_id=fn_id, filetype_id=ftypeid) except StoredFile.DoesNotExist: print('New transfer registered, fn_id {}'.format(fn_id)) file_transferred = StoredFile(rawfile_id=fn_id, filetype_id=ftypeid, servershare=tmpshare, path='', filename=fname, md5='', checked=False) file_transferred.save() jobutil.create_file_job('get_md5', file_transferred.id) else: print('File already registered as transfer, client asks for new ' 'MD5 check after a possible retransfer. Running MD5 check.') jobutil.create_file_job('get_md5', file_transferred.id) finally: return JsonResponse({'fn_id': request.POST['fn_id'], 'state': 'ok'}) else: return HttpResponseNotAllowed(permitted_methods=['POST'])
def upload_userfile_token(request): if request.method != 'POST': return HttpResponseNotAllowed(permitted_methods=['POST']) try: ufile = UserFile.objects.select_related('sfile__servershare', 'upload').get( upload__token=request.POST['token']) except (KeyError, UserFileUpload.DoesNotExist) as e: print(e) return HttpResponseForbidden() else: if ufile.upload.expires < timezone.now(): print('expired', ufile.upload.expires) return HttpResponseForbidden() move_uploaded_file(ufile, request.FILES['file']) jobutil.create_file_job('get_md5', ufile.sfile.id) return HttpResponse()
def mzrefine_file_done(request): """Refined mzML files must get MD5, fn, path and moved to their dataset directory from the analysis output dir (they result from a nextflow analysis run""" # FIXME need to remove the empty dir after moving all the files, how? data = request.POST # create analysis file if ('client_id' not in data or data['client_id'] != settings.ANALYSISCLIENT_APIKEY): return HttpResponseForbidden() sfile = StoredFile.objects.select_related('rawfile__datasetrawfile__dataset').get(pk=data['fn_id']) sfile.path = data['outdir'] sfile.filename = data['filename'] sfile.md5 = data['md5'] sfile.checked = True sfile.save() create_file_job('move_single_file', sfile.id, sfile.rawfile.datasetrawfile.dataset.storage_loc, newname=sfile.filename.split('___')[1]) return HttpResponse()
def manyfile_qc(rawfiles, storedfiles): """For reanalysis or batching by hand""" for rawfn, sfn in zip(rawfiles, storedfiles): try: dsmodels.DatasetRawFile.objects.select_related( 'dataset').filter(rawfile=rawfn).get().dataset except dsmodels.DatasetRawFile.DoesNotExist: dset = add_to_qc(rawfn, sfn) print('Added QC file {} to QC dataset {}'.format(rawfn.id, dset.id)) jobutil.create_file_job('convert_single_mzml', sfn.id) # Do not rerun with the same workflow as previously for rawfn, sfn in zip(rawfiles, storedfiles): if not dashmodels.QCData.objects.filter( analysis__nextflowsearch__nfworkflow=settings.LONGQC_NXF_WF_ID, rawfile=rawfn.id).count(): start_qc_analysis(rawfn, sfn, settings.LONGQC_NXF_WF_ID, settings.LONGQC_FADB_ID) else: print('QC has already been done with this workflow (id: {}) for ' 'rawfile id {}'.format(settings.LONGQC_NXF_WF_ID, rawfn.id))
def do_md5_check(file_transferred): file_registered = file_transferred.rawfile if not file_transferred.md5: return JsonResponse({'fn_id': file_registered.id, 'md5_state': False}) elif file_registered.source_md5 == file_transferred.md5: if not file_transferred.checked: file_transferred.checked = True file_transferred.save() if (not AnalysisResultFile.objects.filter(sfile_id=file_transferred) and SwestoreBackedupFile.objects.filter( storedfile_id=file_transferred.id).count() == 0): fn = file_transferred.filename # FIXME hardcoded instruments are not dynamic! if 'QC' in fn and 'hela' in fn.lower() and any([x in fn for x in ['QE', 'HFLu', 'HFLe', 'Velos', 'HFTo', 'HFGi']]): singlefile_qc(file_transferred.rawfile, file_transferred) jobutil.create_file_job('create_pdc_archive', file_transferred.id, file_transferred.md5) return JsonResponse({'fn_id': file_registered.id, 'md5_state': 'ok'}) else: return JsonResponse({'fn_id': file_registered.id, 'md5_state': 'error'})
def manyfile_qc(rawfiles, storedfiles): """For reanalysis or batching by hand""" for rawfn, sfn in zip(rawfiles, storedfiles): try: dsmodels.DatasetRawFile.objects.select_related('dataset').filter( rawfile=rawfn).get().dataset except dsmodels.DatasetRawFile.DoesNotExist: dset = add_to_qc(rawfn, sfn) print('Added QC file {} to QC dataset {}'.format( rawfn.id, dset.id)) jobutil.create_file_job('convert_single_mzml', sfn.id) # Do not rerun with the same workflow as previously for rawfn, sfn in zip(rawfiles, storedfiles): if not dashmodels.QCData.objects.filter( analysis__nextflowsearch__nfworkflow=settings.LONGQC_NXF_WF_ID, rawfile=rawfn.id).count(): start_qc_analysis(rawfn, sfn, settings.LONGQC_NXF_WF_ID, settings.LONGQC_FADB_ID) else: print('QC has already been done with this workflow (id: {}) for ' 'rawfile id {}'.format(settings.LONGQC_NXF_WF_ID, rawfn.id))
def check_md5_success(request): if not request.method == 'GET': return HttpResponseNotAllowed(permitted_methods=['GET']) try: fn_id = request.GET['fn_id'] ftype = request.GET['ftype'] client_id = request.GET['client_id'] except KeyError: return HttpResponseForbidden() try: check_producer(client_id) except Producer.DoesNotExist: return HttpResponseForbidden() print('Transfer state requested for fn_id {}, type {}'.format( fn_id, ftype)) try: file_transferred = StoredFile.objects.get(rawfile_id=fn_id, filetype=ftype) except StoredFile.DoesNotExist: return JsonResponse({'fn_id': fn_id, 'md5_state': False}) file_registered = file_transferred.rawfile if not file_transferred.md5: return JsonResponse({'fn_id': fn_id, 'md5_state': False}) elif file_registered.source_md5 == file_transferred.md5: if not file_transferred.checked: file_transferred.checked = True file_transferred.save() if (not AnalysisResultFile.objects.filter(sfile_id=file_transferred) and SwestoreBackedupFile.objects.filter( storedfile_id=file_transferred.id).count() == 0): fn = file_transferred.filename if 'QC' in fn and 'hela' in fn.lower() and any( [x in fn for x in ['QE', 'HFLu', 'HFLe', 'Velos']]): singlefile_qc(file_transferred.rawfile, file_transferred) jobutil.create_file_job('create_swestore_backup', file_transferred.id, file_transferred.md5) return JsonResponse({'fn_id': fn_id, 'md5_state': 'ok'}) else: return JsonResponse({'fn_id': fn_id, 'md5_state': 'error'})
def rename_file(request): """Renames a single file. This checks if characters are correct, launches job with bare filename (no extension), since job determines if mutliple files including mzML have to be renamed.""" if not request.method == 'POST': return HttpResponseNotAllowed(permitted_methods=['POST']) data = json.loads(request.body.decode('utf-8')) try: sfile = StoredFile.objects.filter(pk=data['sf_id']).select_related( 'rawfile').get() newfilename = os.path.splitext(data['newname'])[0] #mv_mzml = data['mvmzml'] # TODO optional mzml renaming too? Now it is default except (StoredFile.DoesNotExist, KeyError): print('Stored file to rename does not exist') return HttpResponseForbidden() if request.user.id not in get_file_owners(sfile): print('No ownership of file to rename') return HttpResponseForbidden() if re.match('^[a-zA-Z_0-9\-]*$', newfilename) is None or sfile.filetype_id in [settings.MZML_SFGROUP_ID, settings.REFINEDMZML_SFGROUP_ID]: # TODO Give proper errors to JSON if possible! print('Illegal characters in filename {}'.format(newfilename)) return HttpResponseForbidden() jobutil.create_file_job('rename_file', sfile.id, newfilename)
def set_libraryfile(request): if request.method == 'POST': try: client_id = request.POST['client_id'] fn_id = request.POST['fn_id'] except KeyError as error: print('POST request to register_file with missing parameter, ' '{}'.format(error)) return HttpResponseForbidden() if client_id != settings.ADMIN_APIKEY: print('POST request with incorrect client id ' '{}'.format(client_id)) return HttpResponseForbidden() try: rawfn = RawFile.objects.get(pk=fn_id) except RawFile.DoesNotExist: print('POST request with incorrect fn id ' '{}'.format(fn_id)) return HttpResponseForbidden() else: sfile = StoredFile.objects.select_related('servershare').get( rawfile_id=fn_id) if LibraryFile.objects.filter(sfile__rawfile_id=fn_id): response = {'library': True, 'state': 'ok'} elif sfile.servershare.name == settings.TMPSHARENAME: libfn = LibraryFile.objects.create( sfile=sfile, description=request.POST['desc']) jobutil.create_file_job('move_single_file', sfile.id, settings.LIBRARY_FILE_PATH, newname='libfile_{}_{}'.format( libfn.id, sfile.filename)) response = {'library': True, 'state': 'ok'} else: LibraryFile.objects.create(sfile=sfile, description=request.POST['desc']) response = {'library': False, 'state': 'ok'} return JsonResponse(response)
def set_libraryfile(request): if request.method != 'POST': return HttpResponseNotAllowed(permitted_methods=['POST']) try: client_id = request.POST['client_id'] fn_id = request.POST['fn_id'] except KeyError as error: print('POST request to register_file with missing parameter, ' '{}'.format(error)) return HttpResponseForbidden() if client_id != settings.ADMIN_APIKEY: print('POST request with incorrect client id ' '{}'.format(client_id)) return HttpResponseForbidden() try: rawfn = RawFile.objects.get(pk=fn_id) except RawFile.DoesNotExist: print('POST request with incorrect fn id ' '{}'.format(fn_id)) return HttpResponseForbidden() else: sfile = StoredFile.objects.select_related('servershare').get( rawfile_id=fn_id) if LibraryFile.objects.filter(sfile__rawfile_id=fn_id): response = {'library': True, 'state': 'ok'} elif sfile.servershare.name == settings.TMPSHARENAME: libfn = LibraryFile.objects.create( sfile=sfile, description=request.POST['desc']) jobutil.create_file_job( 'move_single_file', sfile.id, settings.LIBRARY_FILE_PATH, newname='libfile_{}_{}'.format(libfn.id, sfile.filename)) response = {'library': True, 'state': 'ok'} else: LibraryFile.objects.create(sfile=sfile, description=request.POST['desc']) response = {'library': False, 'state': 'ok'} return JsonResponse(response)