def download_datafile_ws(request): if 'url' in request.GET and len(request.GET['url']) > 0: url = urllib.unquote(request.GET['url']) raw_path = url.partition('//')[2] experiment_id = request.GET['experiment_id'] datafile = Dataset_File.objects.filter(url__endswith=raw_path, dataset__experiment__id=experiment_id)[0] if has_datafile_access(request=request, dataset_file_id=datafile.id): file_path = datafile.get_absolute_filepath() try: wrapper = FileWrapper(file(file_path)) response = HttpResponse(wrapper, mimetype=datafile.get_mimetype()) response['Content-Disposition'] = \ 'attachment; filename="%s"' % datafile.filename return response except IOError: return return_response_not_found(request) else: return return_response_not_found(request) else: return return_response_error(request)
def _create_download_response(request, datafile_id, disposition='attachment'): # Get datafile (and return 404 if absent) try: datafile = Dataset_File.objects.get(pk=datafile_id) except Dataset_File.DoesNotExist: return return_response_not_found(request) # Check users has access to datafile if not has_datafile_download_access(request=request, dataset_file_id=datafile.id): return return_response_error(request) # Send an image that can be seen in the browser if disposition == 'inline' and datafile.is_image(): from tardis.tardis_portal.iiif import download_image args = (request, datafile.id, 'full', 'full', '0', 'native') # Send unconverted image if web-compatible if datafile.get_mimetype() in ('image/gif', 'image/jpeg', 'image/png'): return download_image(*args) # Send converted image return download_image(*args, format='png') # Send local file try: # Get file object for datafile file_obj = datafile.get_file() if not file_obj: # If file path doesn't resolve, return not found return return_response_not_found(request) wrapper = FileWrapper(file_obj) response = HttpResponse(wrapper, mimetype=datafile.get_mimetype()) response['Content-Disposition'] = \ '%s; filename="%s"' % (disposition, datafile.filename) return response except IOError: # If we can't read the file, return not found return return_response_not_found(request)
def download_experiment(request, experiment_id, comptype): """ takes string parameter "comptype" for compression method. Currently implemented: "zip" and "tar" """ # TODO: do size estimation, check available temp filespace, check download limits # TODO: intelligent selection of temp file versus in-memory buffering. datafiles = Dataset_File.objects\ .filter(dataset__experiments__id=experiment_id) rootdir = str(experiment_id) msg = _check_download_limits(rootdir, datafiles, comptype) if msg: return return_response_not_found(request) if comptype == "tar": reader = StreamingFile(_write_tar_func(rootdir, datafiles), asynchronous_file_creation=True) response = HttpResponse(FileWrapper(reader), mimetype='application/x-tar') response['Content-Disposition'] = 'attachment; filename="experiment' \ + rootdir + '-complete.tar"' elif comptype == "zip": reader = StreamingFile(_write_zip_func(rootdir, datafiles), asynchronous_file_creation=True) response = HttpResponse(FileWrapper(reader), mimetype='application/zip') response['Content-Disposition'] = 'attachment; filename="experiment' \ + rootdir + '-complete.zip"' else: response = return_response_not_found(request) return response
def _create_download_response(request, datafile_id, disposition='attachment'): # too complex # noqa # Get datafile (and return 404 if absent) try: datafile = DataFile.objects.get(pk=datafile_id) except DataFile.DoesNotExist: return return_response_not_found(request) # Check users has access to datafile if not has_datafile_download_access(request=request, datafile_id=datafile.id): return return_response_error(request) # Send an image that can be seen in the browser if disposition == 'inline' and datafile.is_image(): from tardis.tardis_portal.iiif import download_image args = (request, datafile.id, 'full', 'full', '0', 'native') # Send unconverted image if web-compatible if datafile.get_mimetype() in ('image/gif', 'image/jpeg', 'image/png'): return download_image(*args) # Send converted image return download_image(*args, format='png') # Send local file try: verified_only = True # Query parameter to allow download of unverified files ignore_verif = request.GET.get('ignore_verification_status', '0') # Ensure ignore_verification_status=0 etc works as expected # a bare ?ignore_verification_status is True if ignore_verif.lower() in [u'', u'1', u'true']: verified_only = False # Get file object for datafile file_obj = datafile.get_file(verified_only=verified_only) if not file_obj: # If file path doesn't resolve, return not found if verified_only: return render_error_message(request, "File is unverified, " "please try again later.", status=503) else: return return_response_not_found(request) wrapper = FileWrapper(file_obj, blksize=65535) response = StreamingHttpResponse(wrapper, content_type=datafile.get_mimetype()) response['Content-Disposition'] = \ '%s; filename="%s"' % (disposition, datafile.filename) return response except IOError: # If we can't read the file, return not found return return_response_not_found(request) except ValueError: # raised when replica not verified TODO: custom excptn redirect = request.META.get('HTTP_REFERER', 'http://%s/' % request.META.get('HTTP_HOST')) message = """The file you are trying to access has not yet been verified. Verification is an automated background process. Please try again later or contact the system administrator if the issue persists.""" message = ' '.join(message.split()) # removes spaces redirect = redirect + '#error:' + message return HttpResponseRedirect(redirect)
def download_datafile(request, datafile_id): # todo handle missing file, general error datafile = Dataset_File.objects.get(pk=datafile_id) expid = datafile.dataset.experiment.id if has_datafile_access(request=request, dataset_file_id=datafile.id): url = datafile.url if url.startswith('http://') or url.startswith('https://') \ or url.startswith('ftp://'): return HttpResponseRedirect(datafile.url) elif datafile.protocol == 'tardis': file_path = path.join(settings.FILE_STORE_PATH, str(expid), str(datafile.dataset.id), datafile.filename) if not path.exists(file_path): file_path = path.join(settings.FILE_STORE_PATH, str(expid), datafile.filename) if not path.exists(file_path): logger.error('exp %s: file not found: %s ' % (expid, datafile.url)) return return_response_not_found(request) elif datafile.protocol in ['', 'file']: file_path = datafile.url.partition('://')[2] if not path.exists(file_path): logger.error('exp %s: file not found: %s' % (expid, datafile.url)) return return_response_not_found(request) else: logger.error('exp %s: file protocol unknown: %s' % (expid, datafile.url)) return return_response_not_found(request) else: return return_response_error(request) try: wrapper = FileWrapper(file(file_path)) response = HttpResponse(wrapper, mimetype=datafile.get_mimetype()) response['Content-Disposition'] = \ 'attachment; filename="%s"' % datafile.filename return response except IOError: logger.exception() return return_response_not_found(request)
def download_datafile(request, datafile_id): # todo handle missing file, general error datafile = Dataset_File.objects.get(pk=datafile_id) if has_datafile_access(datafile.id, request.user): url = datafile.url if url.startswith("http://") or url.startswith("https://") or url.startswith("ftp://"): return HttpResponseRedirect(datafile.url) else: file_path = join( settings.FILE_STORE_PATH, str(datafile.dataset.experiment.id), datafile.url.partition("//")[2] ) try: logger.debug(file_path) wrapper = FileWrapper(file(file_path)) response = HttpResponse(wrapper, mimetype="application/octet-stream") response["Content-Disposition"] = 'attachment; filename="' + datafile.filename + '"' # import os # response['Content-Length'] = os.path.getsize(file_path) return response except IOError: return return_response_not_found(request) else: return return_response_error(request)
def download_experiment(request, experiment_id, comptype): """ takes string parameter "comptype" for compression method. Currently implemented: "zip" and "tar" """ # Create the HttpResponse object with the appropriate headers. # TODO: handle no datafile, invalid filename, all http links # (tarfile count?) experiment = Experiment.objects.get(pk=experiment_id) if comptype == "tar": cmd = 'tar -C %s -c %s/' % (path.abspath( settings.FILE_STORE_PATH), str(experiment.id)) # logger.info('TAR COMMAND: ' + cmd) response = HttpResponse(FileWrapper( subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout), mimetype='application/x-tar') response['Content-Disposition'] = 'attachment; filename="experiment' \ + str(experiment.id) + '-complete.tar"' elif comptype == "zip": cmd = 'cd %s; zip -r - %s' % (path.abspath( settings.FILE_STORE_PATH), str(experiment.id)) # logger.info('TAR COMMAND: ' + cmd) response = HttpResponse(FileWrapper( subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout), mimetype='application/zip') response['Content-Disposition'] = 'attachment; filename="experiment' \ + str(experiment.id) + '-complete.zip"' else: return return_response_not_found(request) # response['Content-Length'] = fileSize + 5120 return response
def view_datafile(request, datafile_id): # Get datafile (and return 404 if absent) try: datafile = Dataset_File.objects.get(pk=datafile_id) except Dataset_File.DoesNotExist: return return_response_not_found(request) # Check users has access to datafile if not has_datafile_download_access(request=request, dataset_file_id=datafile.id): return return_response_error(request) # Get actual url for datafile file_url = _get_actual_url(datafile) mimetype = datafile.get_mimetype() if IMAGEMAGICK_AVAILABLE and mimetype in MIMETYPES_TO_VIEW_AS_PNG: with Image(file=urlopen(file_url)) as img: img.format = 'png' content = img.make_blob() # Should show up as a PNG file response = HttpResponse(content, mimetype='image/png') response['Content-Disposition'] = \ 'inline; filename="%s.png"' % datafile.filename else: response = _create_download_response(datafile, file_url, disposition='inline') return response
def download_datafile(request, datafile_id): # todo handle missing file, general error datafile = Dataset_File.objects.get(pk=datafile_id) if has_datafile_access(request=request, dataset_file_id=datafile.id): url = datafile.url if url.startswith('http://') or url.startswith('https://') \ or url.startswith('ftp://'): return HttpResponseRedirect(datafile.url) else: file_path = datafile.get_absolute_filepath() try: wrapper = FileWrapper(file(file_path)) response = HttpResponse(wrapper, mimetype=datafile.get_mimetype()) response['Content-Disposition'] = \ 'attachment; filename="%s"' % datafile.filename return response except IOError: return return_response_not_found(request) else: return return_response_error(request)
def download_experiment(request, experiment_id, comptype): """ takes string parameter "comptype" for compression method. Currently implemented: "zip" and "tar" """ datafiles = Dataset_File.objects\ .filter(dataset__experiments__id=experiment_id) if comptype == "tar": reader = StreamingFile(_write_tar_func(str(experiment_id), datafiles)) response = HttpResponse(FileWrapper(reader), mimetype='application/x-tar') response['Content-Disposition'] = 'attachment; filename="experiment' \ + str(experiment_id) + '-complete.tar"' elif comptype == "zip": reader = StreamingFile(_write_zip_func(str(experiment_id), datafiles)) response = HttpResponse(FileWrapper(reader), mimetype='application/zip') response['Content-Disposition'] = 'attachment; filename="experiment' \ + str(experiment_id) + '-complete.zip"' else: return return_response_not_found(request) # response['Content-Length'] = fileSize + 5120 return response
def download_experiment(request, experiment_id, comptype): """ takes string parameter "comptype" for compression method. Currently implemented: "zip" and "tar" """ # Create the HttpResponse object with the appropriate headers. # TODO: handle no datafile, invalid filename, all http links # (tarfile count?) experiment = Experiment.objects.get(pk=experiment_id) if comptype == "tar": cmd = "tar -C %s -c %s/" % (abspath(settings.FILE_STORE_PATH), str(experiment.id)) # logger.info('TAR COMMAND: ' + cmd) response = HttpResponse( FileWrapper(subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout), mimetype="application/x-tar" ) response["Content-Disposition"] = 'attachment; filename="experiment' + str(experiment.id) + '-complete.tar"' elif comptype == "zip": cmd = "cd %s; zip -r - %s" % (abspath(settings.FILE_STORE_PATH), str(experiment.id)) # logger.info('TAR COMMAND: ' + cmd) response = HttpResponse( FileWrapper(subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout), mimetype="application/zip" ) response["Content-Disposition"] = 'attachment; filename="experiment' + str(experiment.id) + '-complete.zip"' else: return return_response_not_found(request) # response['Content-Length'] = fileSize + 5120 return response
def _create_download_response(request, datafile_id, disposition='attachment'): #import ipdb; ipdb.set_trace() # Get datafile (and return 404 if absent) try: datafile = Dataset_File.objects.get(pk=datafile_id) except Dataset_File.DoesNotExist: return return_response_not_found(request) # Check users has access to datafile if not has_datafile_download_access(request=request, dataset_file_id=datafile.id): return return_response_error(request) # Send an image that can be seen in the browser if disposition == 'inline' and datafile.is_image(): from tardis.tardis_portal.iiif import download_image args = (request, datafile.id, 'full', 'full', '0', 'native') # Send unconverted image if web-compatible if datafile.get_mimetype() in ('image/gif', 'image/jpeg', 'image/png'): return download_image(*args) # Send converted image return download_image(*args, format='png') # Send local file try: # Get file object for datafile file_obj = datafile.get_file() if not file_obj: # If file path doesn't resolve, return not found return return_response_not_found(request) wrapper = FileWrapper(file_obj, blksize=65535) response = StreamingHttpResponse(wrapper, mimetype=datafile.get_mimetype()) response['Content-Disposition'] = \ '%s; filename="%s"' % (disposition, datafile.filename) return response except IOError: # If we can't read the file, return not found return return_response_not_found(request) except ValueError: # raised when replica not verified TODO: custom excptn redirect = request.META.get('HTTP_REFERER', 'http://%s/' % request.META.get('HTTP_HOST')) message = """The file you are trying to access has not yet been verified. Verification is an automated background process. Please try again later or contact the system administrator if the issue persists.""" message = ' '.join(message.split()) # removes spaces redirect = redirect + '#error:' + message return HttpResponseRedirect(redirect)
def _create_download_response(request, datafile_id, disposition='attachment'): #import ipdb; ipdb.set_trace() # Get datafile (and return 404 if absent) try: datafile = Dataset_File.objects.get(pk=datafile_id) except Dataset_File.DoesNotExist: return return_response_not_found(request) # Check users has access to datafile if not has_datafile_download_access(request=request, dataset_file_id=datafile.id): return return_response_error(request) # Send an image that can be seen in the browser if disposition == 'inline' and datafile.is_image(): from tardis.tardis_portal.iiif import download_image args = (request, datafile.id, 'full', 'full', '0', 'native') # Send unconverted image if web-compatible if datafile.get_mimetype() in ('image/gif', 'image/jpeg', 'image/png'): return download_image(*args) # Send converted image return download_image(*args, format='png') # Send local file try: # Get file object for datafile file_obj = datafile.get_file() if not file_obj: # If file path doesn't resolve, return not found return return_response_not_found(request) wrapper = FileWrapper(file_obj) response = HttpResponse(wrapper, mimetype=datafile.get_mimetype()) response['Content-Disposition'] = \ '%s; filename="%s"' % (disposition, datafile.filename) return response except IOError: # If we can't read the file, return not found return return_response_not_found(request) except ValueError: # raised when replica not verified TODO: custom excptn redirect = request.META.get('HTTP_REFERER', 'http://%s/' % request.META.get('HTTP_HOST')) message = """The file you are trying to access has not yet been verified. Verification is an automated background process. Please try again later or contact the system administrator if the issue persists.""" message = ' '.join(message.split()) # removes spaces redirect = redirect + '#error:' + message return HttpResponseRedirect(redirect)
def jobstatus(request, experiment_id): if not request.user.is_authenticated(): return "Not logged in" try: #utils.update_job_status(experiment_id=experiment_id, # user_id=request.user.id) jobs = Job.objects.filter(experiment=Experiment.objects.get( pk=experiment_id)) for job in jobs: job.updateStatus() datasets = jobs.values_list('dataset').distinct() logger.debug(repr(datasets)) disparray = [] for dataset in datasets: dataset = dataset[0] jobids = jobs.filter(dataset=dataset).values_list( 'jobid').distinct() jobidarray = [] for jobid in jobids: finished = True retrieved = True jobid = jobid[0] inttime = uuid.UUID(jobid).time submittime = datetime.datetime.fromtimestamp( (inttime - 0x01b21dd213814000L)*100/1e9) thesejobs = jobs.filter(jobid=jobid) jobdataarray = [] for job in thesejobs: if job.jobstatus.strip() != "Finished": finished = False if job.jobstatus.strip() != "Retrieved": retrieved = False jobdata = { 'status': job.jobstatus, 'hpcjobid': job.hpcjobid, 'submittime': job.submittime, } jobdataarray.append(jobdata) jobiddict = {'jobid': jobid, 'joblist': jobdataarray, 'finished': finished, 'retrieved': retrieved, 'submittime': submittime.strftime( "%d %b %Y, %H:%M:%S")} jobidarray.append(jobiddict) datasetdict = {'dataset': Dataset.objects.get( pk=dataset).description, 'jobidlist': jobidarray} disparray.append(datasetdict) logger.debug(repr(disparray)) c = Context({ #'jobs': jobs, 'disparray': disparray, }) except Experiment.DoesNotExist: return return_response_not_found(request) return render_to_response('mrtardis/jobstatus.html', c)
def download_datafile_ws(request): if 'url' in request.GET and len(request.GET['url']) > 0: url = urllib.unquote(request.GET['url']) raw_path = url.partition('//')[2] experiment_id = request.GET['experiment_id'] datafile = Dataset_File.objects.filter( url__endswith=raw_path, dataset__experiment__id=experiment_id)[0] if has_datafile_access(request=request, dataset_file_id=datafile.id): file_path = datafile.get_absolute_filepath() try: wrapper = FileWrapper(file(file_path)) response = HttpResponse(wrapper, mimetype=datafile.get_mimetype()) response['Content-Disposition'] = \ 'attachment; filename="%s"' % datafile.filename return response except IOError: try: file_path = datafile.get_absolute_filepath_old() wrapper = FileWrapper(file(file_path)) response = HttpResponse(wrapper, mimetype=datafile.get_mimetype()) response['Content-Disposition'] = \ 'attachment; filename="%s"' % datafile.filename return response except IOError: return return_response_not_found(request) else: return return_response_not_found(request) else: return return_response_error(request)
def download_datafile(request, datafile_id): # Get datafile (and return 404 if absent) try: datafile = Dataset_File.objects.get(pk=datafile_id) except Dataset_File.DoesNotExist: return return_response_not_found(request) # Check users has access to datafile if not has_datafile_download_access(request=request, dataset_file_id=datafile.id): return return_response_error(request) # Get actual url for datafile file_url = _get_actual_url(datafile) if not file_url: # If file path doesn't resolve, return not found return return_response_not_found(request) # Just redirect for remote files if not file_url.startswith('file://'): return HttpResponseRedirect(datafile.url) # Send local file try: return _create_download_response(datafile, file_url) except IOError: # If we can't read the file, return not found return return_response_not_found(request)
def download_datafiles(request): # Create the HttpResponse object with the appropriate headers. # TODO: handle no datafile, invalid filename, all http links # (tarfile count?) expid = request.POST['expid'] fileString = '' comptype = "zip" if 'comptype' in request.POST: comptype = request.POST['comptype'] if 'datafile' in request.POST: if len(request.POST.getlist('datafile')) > 500: comptype = "tar" if 'dataset' in request.POST: comptype = "tar" #todo quickfix, calc how many files # the following protocols can be handled by this module protocols = ['', 'file', 'tardis'] known_protocols = len(protocols) if 'datafile' in request.POST or 'dataset' in request.POST: if (len(request.POST.getlist('datafile')) > 0 \ or len(request.POST.getlist('dataset'))) > 0: datasets = request.POST.getlist('dataset') datafiles = request.POST.getlist('datafile') for dsid in datasets: for datafile in Dataset_File.objects.filter(dataset=dsid): if has_datafile_access(request=request, dataset_file_id=datafile.id): p = datafile.protocol if not p in protocols: protocols += [p] absolute_filename = datafile.url.partition('//')[2] if (datafile.url.partition('//')[0] == 'tardis:'): #temp fix for old data filepath = '%s/%s/%s' %\ (expid, str(datafile.dataset.id), absolute_filename) print filepath + "######" try: wrapper = FileWrapper(file( datafile.get_absolute_filepath()))\ #exists test. os.exists broken except IOError: print "OLD FILE DETECTED" filepath = '%s/%s' % (expid, absolute_filename) fileString += ('\"' + filepath + '\" ') print fileString else: fileString += '\"%s/%s\" ' %\ (expid, absolute_filename) for dfid in datafiles: datafile = Dataset_File.objects.get(pk=dfid) if datafile.dataset.id in datasets: continue if has_datafile_access(request=request, dataset_file_id=datafile.id): p = datafile.protocol if not p in protocols: protocols += [p] absolute_filename = datafile.url.partition('//')[2] if (datafile.url.partition('//')[0] == 'tardis:'): #temp fix for old data filepath = '\"%s/%s/%s\" ' %\ (expid, str(datafile.dataset.id), absolute_filename) print filepath + "######" try: wrapper = FileWrapper(file( datafile.get_absolute_filepath()))\ #exists test. os.exists broken except IOError: print "OLD FILE DETECTED" filepath = '\"%s/%s\" ' %\ (expid, absolute_filename) fileString += filepath print fileString else: fileString += '\"%s/%s\" ' % (expid, absolute_filename) else: return return_response_not_found(request) elif 'url' in request.POST: if not len(request.POST.getlist('url')) == 0: comptype = "tar" #todo quickfix for zip error fileString = "" for url in request.POST.getlist('url'): url = urllib.unquote(url) raw_path = url.partition('//')[2] experiment_id = request.POST['expid'] datafile = Dataset_File.objects.filter( url__endswith=raw_path, dataset__experiment__id=experiment_id)[0] if has_datafile_access(request=request, dataset_file_id=datafile.id): p = datafile.protocol if not p in protocols: protocols += [p] absolute_filename = datafile.url.partition('//')[2] if (datafile.url.partition('//')[0] == 'tardis:'): # expects tardis: formatted stuff # to not include dataset id #temp fix for old data filepath = '\"%s/%s/%s\" ' %\ (expid, str(datafile.dataset.id), absolute_filename) print filepath + "######" try: wrapper = FileWrapper(file( datafile.get_absolute_filepath()))\ #exists test. os.exists broken except IOError: print "OLD FILE DETECTED" filepath = '\"%s/%s\" ' %\ (expid, absolute_filename) fileString += ('\"' + filepath + '\" ') print fileString else: fileString += '\"%s/%s\" ' % (expid, absolute_filename) else: return return_response_not_found(request) else: return return_response_not_found(request) # more than one external download location? if len(protocols) > known_protocols + 2: response = HttpResponseNotFound() response.write('<p>Different locations selected!</p>\n') response.write('Please limit your selection and try again.\n') return response # redirect request if another (external) download protocol was found elif len(protocols) == known_protocols + 1: from django.core.urlresolvers import reverse, resolve try: for module in settings.DOWNLOAD_PROVIDERS: if module[0] == protocols[3]: url = reverse('%s.download_datafiles' % module[1]) view, args, kwargs = resolve(url) kwargs['request'] = request return view(*args, **kwargs) except: return return_response_not_found(request) else: # tarfile class doesn't work on large files being added and # streamed on the fly, so going command-line-o if not fileString: return return_response_error(request) if comptype == "tar": cmd = 'tar -C %s -c %s' % (settings.FILE_STORE_PATH, fileString) # logger.info(cmd) response = \ HttpResponse(FileWrapper(subprocess.Popen( cmd, stdout=subprocess.PIPE, shell=True).stdout), mimetype='application/x-tar') response['Content-Disposition'] = \ 'attachment; filename="experiment%s-selection.tar"' % expid return response else: cmd = 'cd %s; zip -r - %s' % (settings.FILE_STORE_PATH, fileString) # logger.info(cmd) response = \ HttpResponse(FileWrapper(subprocess.Popen( cmd, stdout=subprocess.PIPE, shell=True).stdout), mimetype='application/zip') response['Content-Disposition'] = \ 'attachment; filename="experiment%s-selection.zip"' % expid return response
def download_datafiles(request): # Create the HttpResponse object with the appropriate headers. # TODO: handle no datafile, invalid filename, all http links # (tarfile count?) comptype = "zip" if 'comptype' in request.POST: comptype = request.POST['comptype'] if 'datafile' in request.POST or 'dataset' in request.POST: if (len(request.POST.getlist('datafile')) > 0 \ or len(request.POST.getlist('dataset'))) > 0: datasets = request.POST.getlist('dataset') datafiles = request.POST.getlist('datafile') # Generator to produce datafiles from dataset id def get_dataset_datafiles(dsid): for datafile in Dataset_File.objects.filter(dataset=dsid): if has_datafile_download_access( request=request, dataset_file_id=datafile.id): yield datafile # Generator to produce datafile from datafile id def get_datafile(dfid): datafile = Dataset_File.objects.get(pk=dfid) if has_datafile_download_access(request=request, dataset_file_id=datafile.id): yield datafile # Take chained generators and turn them into a set of datafiles df_set = set( chain( chain.from_iterable(map(get_dataset_datafiles, datasets)), chain.from_iterable(map(get_datafile, datafiles)))) else: return return_response_not_found(request) elif 'url' in request.POST: if not len(request.POST.getlist('url')) == 0: return return_response_not_found(request) for url in request.POST.getlist('url'): url = urllib.unquote(url) raw_path = url.partition('//')[2] experiment_id = request.POST['expid'] datafile = Dataset_File.objects.filter( url__endswith=raw_path, dataset__experiment__id=experiment_id)[0] if has_datafile_download_access(request=request, dataset_file_id=datafile.id): df_set = set([datafile]) else: return return_response_not_found(request) logger.info('Files for archive command: %s' % df_set) if len(df_set) == 0: return return_response_error(request) # Handle missing experiment ID - only need it for naming try: expid = request.POST['expid'] except KeyError: expid = iter(df_set).next().dataset.get_first_experiment().id if comptype == "tar": reader = StreamingFile(_write_tar_func('datasets', df_set)) # logger.info(cmd) response = \ HttpResponse(FileWrapper(reader), mimetype='application/x-tar') response['Content-Disposition'] = \ 'attachment; filename="experiment%s-selection.tar"' % expid return response else: reader = StreamingFile(_write_zip_func('datasets', df_set)) # logger.info(cmd) response = \ HttpResponse(FileWrapper(reader), mimetype='application/zip') response['Content-Disposition'] = \ 'attachment; filename="experiment%s-selection.zip"' % expid return response
def download_datafiles(request): # Create the HttpResponse object with the appropriate headers. # TODO: handle no datafile, invalid filename, all http links # (tarfile count?) expid = request.POST["expid"] protocols = [] fileString = "" fileSize = 0 if "datafile" or "dataset" in request.POST: if (len(request.POST.getlist("datafile")) > 0 or len(request.POST.getlist("dataset"))) > 0: datasets = request.POST.getlist("dataset") datafiles = request.POST.getlist("datafile") for dsid in datasets: for datafile in Dataset_File.objects.filter(dataset=dsid): if has_datafile_access(datafile.id, request.user): p = datafile.protocol if not p in protocols: protocols += [p] if datafile.url.startswith("file://"): absolute_filename = datafile.url.partition("//")[2] fileString += expid + "/" + absolute_filename + " " fileSize += long(datafile.size) for dfid in datafiles: datafile = Dataset_File.objects.get(pk=dfid) if datafile.dataset.id in datasets: continue if has_datafile_access(dfid, request.user): p = datafile.protocol if not p in protocols: protocols += [p] if datafile.url.startswith("file://"): absolute_filename = datafile.url.partition("//")[2] fileString += expid + "/" + absolute_filename + " " fileSize += long(datafile.size) else: return return_response_not_found(request) elif "url" in request.POST: if not len(request.POST.getlist("url")) == 0: for url in request.POST.getlist("url"): datafile = Dataset_File.objects.get( url=urllib.unquote(url), dataset__experiment__id=request.POST["expid"] ) if has_datafile_access(datafile.id, request.user): p = datafile.protocol if not p in protocols: protocols += [p] if datafile.url.startswith("file://"): absolute_filename = datafile.url.partition("//")[2] fileString += expid + "/" + absolute_filename + " " fileSize += long(datafile.size) else: return return_response_not_found(request) else: return return_response_not_found(request) # more than one download location? if len(protocols) > 1: response = HttpResponseNotFound() response.write("<p>Different locations selected!</p>\n") response.write("Please limit your selection and try again.\n") return response # redirect request if download protocol found if protocols[0] != "": from django.core.urlresolvers import resolve view, args, kwargs = resolve("/%s%s" % (protocols[0], request.path)) kwargs["request"] = request return view(*args, **kwargs) else: # tarfile class doesn't work on large files being added and # streamed on the fly, so going command-line-o cmd = "tar -C %s -c %s" % (settings.FILE_STORE_PATH, fileString) response = HttpResponse( FileWrapper(subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout), mimetype="application/x-tar" ) response["Content-Disposition"] = "attachment; filename=experiment%s.tar" % expid response["Content-Length"] = fileSize + 5120 return response
def download_datafiles(request): # Create the HttpResponse object with the appropriate headers. # TODO: handle no datafile, invalid filename, all http links # (tarfile count?) expid = request.POST['expid'] fileString = '' comptype = "zip" if 'comptype' in request.POST: comptype = request.POST['comptype'] if 'datafile' in request.POST: if len(request.POST.getlist('datafile')) > 500: comptype = "tar" if 'dataset' in request.POST: comptype = "tar" #todo quickfix, calc how many files # the following protocols can be handled by this module protocols = ['', 'file', 'tardis'] known_protocols = len(protocols) if 'datafile' in request.POST or 'dataset' in request.POST: if (len(request.POST.getlist('datafile')) > 0 \ or len(request.POST.getlist('dataset'))) > 0: datasets = request.POST.getlist('dataset') datafiles = request.POST.getlist('datafile') for dsid in datasets: for datafile in Dataset_File.objects.filter(dataset=dsid): if has_datafile_access(request=request, dataset_file_id=datafile.id): p = datafile.protocol if not p in protocols: protocols += [p] absolute_filename = datafile.url.partition('//')[2] if(datafile.url.partition('//')[0] == 'tardis:'): #temp fix for old data filepath = '%s/%s/%s' %\ (expid, str(datafile.dataset.id), absolute_filename) print filepath + "######" try: wrapper = FileWrapper(file( datafile.get_absolute_filepath()))\ #exists test. os.exists broken except IOError: print "OLD FILE DETECTED" filepath = '%s/%s' % (expid, absolute_filename) fileString += ('\"' + filepath + '\" ') print fileString else: fileString += '\"%s/%s\" ' %\ (expid, absolute_filename) for dfid in datafiles: datafile = Dataset_File.objects.get(pk=dfid) if datafile.dataset.id in datasets: continue if has_datafile_access(request=request, dataset_file_id=datafile.id): p = datafile.protocol if not p in protocols: protocols += [p] absolute_filename = datafile.url.partition('//')[2] if(datafile.url.partition('//')[0] == 'tardis:'): #temp fix for old data filepath = '\"%s/%s/%s\" ' %\ (expid, str(datafile.dataset.id), absolute_filename) print filepath + "######" try: wrapper = FileWrapper(file( datafile.get_absolute_filepath()))\ #exists test. os.exists broken except IOError: print "OLD FILE DETECTED" filepath = '\"%s/%s\" ' %\ (expid, absolute_filename) fileString += filepath print fileString else: fileString += '\"%s/%s\" ' % (expid, absolute_filename) else: return return_response_not_found(request) elif 'url' in request.POST: if not len(request.POST.getlist('url')) == 0: comptype = "tar" #todo quickfix for zip error fileString = "" for url in request.POST.getlist('url'): url = urllib.unquote(url) raw_path = url.partition('//')[2] experiment_id = request.POST['expid'] datafile = Dataset_File.objects.filter(url__endswith=raw_path, dataset__experiment__id=experiment_id)[0] if has_datafile_access(request=request, dataset_file_id=datafile.id): p = datafile.protocol if not p in protocols: protocols += [p] absolute_filename = datafile.url.partition('//')[2] if(datafile.url.partition('//')[0] == 'tardis:'): # expects tardis: formatted stuff # to not include dataset id #temp fix for old data filepath = '\"%s/%s/%s\" ' %\ (expid, str(datafile.dataset.id), absolute_filename) print filepath + "######" try: wrapper = FileWrapper(file( datafile.get_absolute_filepath()))\ #exists test. os.exists broken except IOError: print "OLD FILE DETECTED" filepath = '\"%s/%s\" ' %\ (expid, absolute_filename) fileString += ('\"' + filepath + '\" ') print fileString else: fileString += '\"%s/%s\" ' % (expid, absolute_filename) else: return return_response_not_found(request) else: return return_response_not_found(request) # more than one external download location? if len(protocols) > known_protocols + 2: response = HttpResponseNotFound() response.write('<p>Different locations selected!</p>\n') response.write('Please limit your selection and try again.\n') return response # redirect request if another (external) download protocol was found elif len(protocols) == known_protocols + 1: from django.core.urlresolvers import reverse, resolve try: for module in settings.DOWNLOAD_PROVIDERS: if module[0] == protocols[3]: url = reverse('%s.download_datafiles' % module[1]) view, args, kwargs = resolve(url) kwargs['request'] = request return view(*args, **kwargs) except: return return_response_not_found(request) else: # tarfile class doesn't work on large files being added and # streamed on the fly, so going command-line-o if not fileString: return return_response_error(request) if comptype == "tar": cmd = 'tar -C %s -c %s' % (settings.FILE_STORE_PATH, fileString) # logger.info(cmd) response = \ HttpResponse(FileWrapper(subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=open(devnull, 'w'), shell=True).stdout), mimetype='application/x-tar') response['Content-Disposition'] = \ 'attachment; filename="experiment%s-selection.tar"' % expid return response else: cmd = 'cd %s; zip -r - %s' % (settings.FILE_STORE_PATH, fileString) # logger.info(cmd) response = \ HttpResponse(FileWrapper(subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=open(devnull, 'w'), shell=True).stdout), mimetype='application/zip') response['Content-Disposition'] = \ 'attachment; filename="experiment%s-selection.zip"' % expid return response
def download_datafiles(request): # Create the HttpResponse object with the appropriate headers. # TODO: handle no datafile, invalid filename, all http links # (tarfile count?) expid = request.POST["expid"] fileString = "" fileSize = 0 comptype = "zip" if "comtype" in request.POST: comptype = request.POST["comptype"] # the following protocols can be handled by this module protocols = ["", "file", "tardis"] known_protocols = len(protocols) if "datafile" or "dataset" in request.POST: if (len(request.POST.getlist("datafile")) > 0 or len(request.POST.getlist("dataset"))) > 0: datasets = request.POST.getlist("dataset") datafiles = request.POST.getlist("datafile") for dsid in datasets: for datafile in Dataset_File.objects.filter(dataset=dsid): if has_datafile_access(request=request, dataset_file_id=datafile.id): p = datafile.protocol if not p in protocols: protocols += [p] absolute_filename = datafile.url.partition("//")[2] if datafile.url.partition("//")[0] == "tardis:": fileString += "%s/%s/%s " % (expid, str(datafile.dataset.id), absolute_filename) else: fileString += "%s/%s " % (expid, absolute_filename) fileSize += long(datafile.size) for dfid in datafiles: datafile = Dataset_File.objects.get(pk=dfid) if datafile.dataset.id in datasets: continue if has_datafile_access(request=request, dataset_file_id=datafile.id): p = datafile.protocol if not p in protocols: protocols += [p] absolute_filename = datafile.url.partition("//")[2] if datafile.url.partition("//")[0] == "tardis:": fileString += "%s/%s/%s " % (expid, str(datafile.dataset.id), absolute_filename) else: fileString += "%s/%s " % (expid, absolute_filename) fileSize += long(datafile.size) else: return return_response_not_found(request) elif "url" in request.POST: if not len(request.POST.getlist("url")) == 0: fileString = "" fileSize = 0 for url in request.POST.getlist("url"): url = urllib.unquote(url) raw_path = url.partition("//")[2] experiment_id = request.POST["expid"] datafile = Dataset_File.objects.filter(url__endswith=raw_path, dataset__experiment__id=experiment_id)[0] if has_datafile_access(request=request, dataset_file_id=datafile.id): p = datafile.protocol if not p in protocols: protocols += [p] absolute_filename = datafile.url.partition("//")[2] if datafile.url.partition("//")[0] == "tardis:": # expects tardis: formatted stuff to not include dataset id fileString += "%s/%s/%s " % (expid, str(datafile.dataset.id), absolute_filename) else: fileString += "%s/%s " % (expid, absolute_filename) fileSize += long(datafile.size) else: return return_response_not_found(request) else: return return_response_not_found(request) # more than one external download location? if len(protocols) > known_protocols + 2: response = HttpResponseNotFound() response.write("<p>Different locations selected!</p>\n") response.write("Please limit your selection and try again.\n") return response # redirect request if another (external) download protocol was found elif len(protocols) == known_protocols + 1: from django.core.urlresolvers import reverse, resolve try: for module in settings.DOWNLOAD_PROVIDERS: if module[0] == protocols[3]: url = reverse("%s.download_datafiles" % module[1]) view, args, kwargs = resolve(url) kwargs["request"] = request return view(*args, **kwargs) except: return return_response_not_found(request) else: # tarfile class doesn't work on large files being added and # streamed on the fly, so going command-line-o if not fileString: return return_response_error(request) if comptype == "tar": cmd = "tar -C %s -c %s" % (settings.FILE_STORE_PATH, fileString) # logger.info(cmd) response = HttpResponse( FileWrapper(subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout), mimetype="application/x-tar", ) response["Content-Disposition"] = 'attachment; filename="experiment%s-selection.tar"' % expid response["Content-Length"] = fileSize + 5120 return response else: cmd = "cd %s; zip -r - %s" % (settings.FILE_STORE_PATH, fileString) # logger.info(cmd) response = HttpResponse( FileWrapper(subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout), mimetype="application/zip", ) response["Content-Disposition"] = 'attachment; filename="experiment%s-selection.zip"' % expid response["Content-Length"] = fileSize + 5120 return response
def download_datafiles(request): # Create the HttpResponse object with the appropriate headers. # TODO: handle no datafile, invalid filename, all http links # (tarfile count?) expid = request.POST['expid'] protocols = [] fileString = '' fileSize = 0 # the following protocols can be handled by this module protocols = ['', 'file', 'tardis'] if 'datafile' or 'dataset' in request.POST: if (len(request.POST.getlist('datafile')) > 0 \ or len(request.POST.getlist('dataset'))) > 0: datasets = request.POST.getlist('dataset') datafiles = request.POST.getlist('datafile') for dsid in datasets: for datafile in Dataset_File.objects.filter(dataset=dsid): if has_datafile_access(request=request, dataset_file_id=datafile.id): p = datafile.protocol if not p in protocols: protocols += [p] absolute_filename = datafile.url.partition('//')[2] fileString += '%s/%s ' % (expid, absolute_filename) fileSize += long(datafile.size) for dfid in datafiles: datafile = Dataset_File.objects.get(pk=dfid) if datafile.dataset.id in datasets: continue if has_datafile_access(request=request, dataset_file_id=datafile.id): p = datafile.protocol if not p in protocols: protocols += [p] absolute_filename = datafile.url.partition('//')[2] fileString += '%s/%s ' % (expid, absolute_filename) fileSize += long(datafile.size) else: return return_response_not_found(request) # TODO: check if we really still need this method elif 'url' in request.POST: if not len(request.POST.getlist('url')) == 0: for url in request.POST.getlist('url'): datafile = \ Dataset_File.objects.get(url=urllib.unquote(url), dataset__experiment__id=request.POST['expid']) if has_datafile_access(request=request, dataset_file_id=datafile.id): p = datafile.protocol if not p in protocols: protocols += [p] absolute_filename = datafile.url.partition('//')[2] fileString += '%s/%s ' % (expid, absolute_filename) fileSize += long(datafile.size) else: return return_response_not_found(request) else: return return_response_not_found(request) # more than one external download location? if len(protocols) > 4: response = HttpResponseNotFound() response.write('<p>Different locations selected!</p>\n') response.write('Please limit your selection and try again.\n') return response # redirect request if another (external) download protocol was found elif len(protocols) == 4: from django.core.urlresolvers import resolve view, args, kwargs = resolve('/%s%s' % (protocols[4], request.path)) kwargs['request'] = request return view(*args, **kwargs) else: # tarfile class doesn't work on large files being added and # streamed on the fly, so going command-line-o if not fileString: return return_response_error(request) cmd = 'tar -C %s -c %s' % (settings.FILE_STORE_PATH, fileString) # logger.info(cmd) response = \ HttpResponse(FileWrapper(subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout), mimetype='application/x-tar') response['Content-Disposition'] = \ 'attachment; filename="experiment%s.tar"' % expid response['Content-Length'] = fileSize + 5120 return response
def download_datafiles(request): # Create the HttpResponse object with the appropriate headers. # TODO: handle no datafile, invalid filename, all http links # (tarfile count?) comptype = "zip" if 'comptype' in request.POST: comptype = request.POST['comptype'] if 'datafile' in request.POST or 'dataset' in request.POST: if (len(request.POST.getlist('datafile')) > 0 \ or len(request.POST.getlist('dataset'))) > 0: datasets = request.POST.getlist('dataset') datafiles = request.POST.getlist('datafile') # Generator to produce datafiles from dataset id def get_dataset_datafiles(dsid): for datafile in Dataset_File.objects.filter(dataset=dsid): if has_datafile_download_access(request=request, dataset_file_id=datafile.id): yield datafile # Generator to produce datafile from datafile id def get_datafile(dfid): datafile = Dataset_File.objects.get(pk=dfid) if has_datafile_download_access(request=request, dataset_file_id=datafile.id): yield datafile # Take chained generators and turn them into a set of datafiles df_set = set(chain(chain.from_iterable(map(get_dataset_datafiles, datasets)), chain.from_iterable(map(get_datafile, datafiles)))) else: return return_response_not_found(request) elif 'url' in request.POST: if not len(request.POST.getlist('url')) == 0: return return_response_not_found(request) for url in request.POST.getlist('url'): url = urllib.unquote(url) raw_path = url.partition('//')[2] experiment_id = request.POST['expid'] datafile = Dataset_File.objects.filter(url__endswith=raw_path, dataset__experiment__id=experiment_id)[0] if has_datafile_download_access(request=request, dataset_file_id=datafile.id): df_set = set([datafile]) else: return return_response_not_found(request) logger.info('Files for archive command: %s' % df_set) if len(df_set) == 0: return return_response_error(request) # Handle missing experiment ID - only need it for naming try: expid = request.POST['expid'] except KeyError: expid = iter(df_set).next().dataset.get_first_experiment().id if comptype == "tar": reader = StreamingFile(_write_tar_func('datasets', df_set)) # logger.info(cmd) response = \ HttpResponse(FileWrapper(reader), mimetype='application/x-tar') response['Content-Disposition'] = \ 'attachment; filename="experiment%s-selection.tar"' % expid return response else: reader = StreamingFile(_write_zip_func('datasets', df_set)) # logger.info(cmd) response = \ HttpResponse(FileWrapper(reader), mimetype='application/zip') response['Content-Disposition'] = \ 'attachment; filename="experiment%s-selection.zip"' % expid return response
def download_datafile(request, datafile_id): # todo handle missing file, general error datafile = Dataset_File.objects.get(pk=datafile_id) expid = datafile.dataset.experiment.id if has_datafile_access(request=request, dataset_file_id=datafile.id): url = datafile.url if url.startswith('http://') or url.startswith('https://') \ or url.startswith('ftp://'): return HttpResponseRedirect(datafile.url) elif datafile.protocol == 'tardis' or \ datafile.url.startswith('tardis'): raw_path = url.partition('//')[2] file_path = path.join(settings.FILE_STORE_PATH, str(expid), str(datafile.dataset.id), raw_path) try: wrapper = FileWrapper(file(file_path)) response = HttpResponse(wrapper, mimetype=datafile.get_mimetype()) response['Content-Disposition'] = \ 'attachment; filename="%s"' % datafile.filename return response except IOError: try: file_path = path.join(settings.FILE_STORE_PATH, str(expid), raw_path) print file_path wrapper = FileWrapper(file(file_path)) response = HttpResponse(wrapper, mimetype=datafile.get_mimetype()) response['Content-Disposition'] = \ 'attachment; filename="%s"' % datafile.filename return response except IOError: return return_response_not_found(request) elif datafile.protocol in ['', 'file']: file_path = datafile.url.partition('://')[2] if not path.exists(file_path): return return_response_not_found(request) else: # logger.error('exp %s: file protocol unknown: %s' % (expid, # datafile.url)) return return_response_not_found(request) else: return return_response_error(request) try: wrapper = FileWrapper(file(file_path)) response = HttpResponse(wrapper, mimetype=datafile.get_mimetype()) response['Content-Disposition'] = \ 'attachment; filename="%s"' % datafile.filename return response except IOError: logger.exception() return return_response_not_found(request)