Example #1
0
def download_datafile_ws(request):
    if 'url' in request.GET and len(request.GET['url']) > 0:
        url = urllib.unquote(request.GET['url'])
        raw_path = url.partition('//')[2]
        experiment_id = request.GET['experiment_id']
        datafile = Dataset_File.objects.filter(url__endswith=raw_path, dataset__experiment__id=experiment_id)[0]

        if has_datafile_access(request=request,
                               dataset_file_id=datafile.id):

            file_path = datafile.get_absolute_filepath()

            try:
                wrapper = FileWrapper(file(file_path))
                response = HttpResponse(wrapper,
                                        mimetype=datafile.get_mimetype())
                response['Content-Disposition'] = \
                    'attachment; filename="%s"' % datafile.filename
                return response

            except IOError:
                return return_response_not_found(request)

        else:
            return return_response_not_found(request)

    else:
        return return_response_error(request)
Example #2
0
def _create_download_response(request, datafile_id, disposition='attachment'):
    # Get datafile (and return 404 if absent)
    try:
        datafile = Dataset_File.objects.get(pk=datafile_id)
    except Dataset_File.DoesNotExist:
        return return_response_not_found(request)
    # Check users has access to datafile
    if not has_datafile_download_access(request=request,
                                        dataset_file_id=datafile.id):
        return return_response_error(request)
    # Send an image that can be seen in the browser
    if disposition == 'inline' and datafile.is_image():
        from tardis.tardis_portal.iiif import download_image
        args = (request, datafile.id, 'full', 'full', '0', 'native')
        # Send unconverted image if web-compatible
        if datafile.get_mimetype() in ('image/gif', 'image/jpeg', 'image/png'):
            return download_image(*args)
        # Send converted image
        return download_image(*args, format='png')
    # Send local file
    try:
        # Get file object for datafile
        file_obj = datafile.get_file()
        if not file_obj:
            # If file path doesn't resolve, return not found
            return return_response_not_found(request)
        wrapper = FileWrapper(file_obj)
        response = HttpResponse(wrapper,
                                mimetype=datafile.get_mimetype())
        response['Content-Disposition'] = \
            '%s; filename="%s"' % (disposition, datafile.filename)
        return response
    except IOError:
        # If we can't read the file, return not found
        return return_response_not_found(request)
def download_experiment(request, experiment_id, comptype):
    """
    takes string parameter "comptype" for compression method.
    Currently implemented: "zip" and "tar"
    """
    # TODO: do size estimation, check available temp filespace, check download limits 
    # TODO: intelligent selection of temp file versus in-memory buffering.
    datafiles = Dataset_File.objects\
        .filter(dataset__experiments__id=experiment_id)
    rootdir = str(experiment_id)
    msg = _check_download_limits(rootdir, datafiles, comptype)
    if msg:
        return return_response_not_found(request)

    if comptype == "tar":
        reader = StreamingFile(_write_tar_func(rootdir, datafiles),
                               asynchronous_file_creation=True)

        response = HttpResponse(FileWrapper(reader),
                                mimetype='application/x-tar')
        response['Content-Disposition'] = 'attachment; filename="experiment' \
            + rootdir + '-complete.tar"'
    elif comptype == "zip":
        reader = StreamingFile(_write_zip_func(rootdir, datafiles),
                               asynchronous_file_creation=True)
        response = HttpResponse(FileWrapper(reader),
                                mimetype='application/zip')

        response['Content-Disposition'] = 'attachment; filename="experiment' \
            + rootdir + '-complete.zip"'
    else:
        response = return_response_not_found(request)
    return response
Example #4
0
def _create_download_response(request, datafile_id, disposition='attachment'):  # too complex # noqa
    # Get datafile (and return 404 if absent)
    try:
        datafile = DataFile.objects.get(pk=datafile_id)
    except DataFile.DoesNotExist:
        return return_response_not_found(request)
    # Check users has access to datafile
    if not has_datafile_download_access(request=request,
                                        datafile_id=datafile.id):
        return return_response_error(request)
    # Send an image that can be seen in the browser
    if disposition == 'inline' and datafile.is_image():
        from tardis.tardis_portal.iiif import download_image
        args = (request, datafile.id, 'full', 'full', '0', 'native')
        # Send unconverted image if web-compatible
        if datafile.get_mimetype() in ('image/gif', 'image/jpeg', 'image/png'):
            return download_image(*args)
        # Send converted image
        return download_image(*args, format='png')
    # Send local file
    try:
        verified_only = True
        # Query parameter to allow download of unverified files
        ignore_verif = request.GET.get('ignore_verification_status', '0')
        # Ensure ignore_verification_status=0 etc works as expected
        # a bare ?ignore_verification_status is True
        if ignore_verif.lower() in [u'', u'1', u'true']:
            verified_only = False

        # Get file object for datafile
        file_obj = datafile.get_file(verified_only=verified_only)
        if not file_obj:
            # If file path doesn't resolve, return not found
            if verified_only:
                return render_error_message(request,
                                            "File is unverified, "
                                            "please try again later.",
                                            status=503)
            else:
                return return_response_not_found(request)
        wrapper = FileWrapper(file_obj, blksize=65535)
        response = StreamingHttpResponse(wrapper,
                                         content_type=datafile.get_mimetype())
        response['Content-Disposition'] = \
            '%s; filename="%s"' % (disposition, datafile.filename)
        return response
    except IOError:
        # If we can't read the file, return not found
        return return_response_not_found(request)
    except ValueError:  # raised when replica not verified TODO: custom excptn
        redirect = request.META.get('HTTP_REFERER',
                                    'http://%s/' %
                                    request.META.get('HTTP_HOST'))
        message = """The file you are trying to access has not yet been
                     verified. Verification is an automated background process.
                     Please try again later or contact the system
                     administrator if the issue persists."""
        message = ' '.join(message.split())  # removes spaces
        redirect = redirect + '#error:' + message
        return HttpResponseRedirect(redirect)
Example #5
0
def _create_download_response(request, datafile_id, disposition='attachment'):  # too complex # noqa
    # Get datafile (and return 404 if absent)
    try:
        datafile = DataFile.objects.get(pk=datafile_id)
    except DataFile.DoesNotExist:
        return return_response_not_found(request)
    # Check users has access to datafile
    if not has_datafile_download_access(request=request,
                                        datafile_id=datafile.id):
        return return_response_error(request)
    # Send an image that can be seen in the browser
    if disposition == 'inline' and datafile.is_image():
        from tardis.tardis_portal.iiif import download_image
        args = (request, datafile.id, 'full', 'full', '0', 'native')
        # Send unconverted image if web-compatible
        if datafile.get_mimetype() in ('image/gif', 'image/jpeg', 'image/png'):
            return download_image(*args)
        # Send converted image
        return download_image(*args, format='png')
    # Send local file
    try:
        verified_only = True
        # Query parameter to allow download of unverified files
        ignore_verif = request.GET.get('ignore_verification_status', '0')
        # Ensure ignore_verification_status=0 etc works as expected
        # a bare ?ignore_verification_status is True
        if ignore_verif.lower() in [u'', u'1', u'true']:
            verified_only = False

        # Get file object for datafile
        file_obj = datafile.get_file(verified_only=verified_only)
        if not file_obj:
            # If file path doesn't resolve, return not found
            if verified_only:
                return render_error_message(request,
                                            "File is unverified, "
                                            "please try again later.",
                                            status=503)
            else:
                return return_response_not_found(request)
        wrapper = FileWrapper(file_obj, blksize=65535)
        response = StreamingHttpResponse(wrapper,
                                         content_type=datafile.get_mimetype())
        response['Content-Disposition'] = \
            '%s; filename="%s"' % (disposition, datafile.filename)
        return response
    except IOError:
        # If we can't read the file, return not found
        return return_response_not_found(request)
    except ValueError:  # raised when replica not verified TODO: custom excptn
        redirect = request.META.get('HTTP_REFERER',
                                    'http://%s/' %
                                    request.META.get('HTTP_HOST'))
        message = """The file you are trying to access has not yet been
                     verified. Verification is an automated background process.
                     Please try again later or contact the system
                     administrator if the issue persists."""
        message = ' '.join(message.split())  # removes spaces
        redirect = redirect + '#error:' + message
        return HttpResponseRedirect(redirect)
Example #6
0
def _create_download_response(request, datafile_id, disposition='attachment'):
    # Get datafile (and return 404 if absent)
    try:
        datafile = Dataset_File.objects.get(pk=datafile_id)
    except Dataset_File.DoesNotExist:
        return return_response_not_found(request)
    # Check users has access to datafile
    if not has_datafile_download_access(request=request,
                                        dataset_file_id=datafile.id):
        return return_response_error(request)
    # Send an image that can be seen in the browser
    if disposition == 'inline' and datafile.is_image():
        from tardis.tardis_portal.iiif import download_image
        args = (request, datafile.id, 'full', 'full', '0', 'native')
        # Send unconverted image if web-compatible
        if datafile.get_mimetype() in ('image/gif', 'image/jpeg', 'image/png'):
            return download_image(*args)
        # Send converted image
        return download_image(*args, format='png')
    # Send local file
    try:
        # Get file object for datafile
        file_obj = datafile.get_file()
        if not file_obj:
            # If file path doesn't resolve, return not found
            return return_response_not_found(request)
        wrapper = FileWrapper(file_obj)
        response = HttpResponse(wrapper, mimetype=datafile.get_mimetype())
        response['Content-Disposition'] = \
            '%s; filename="%s"' % (disposition, datafile.filename)
        return response
    except IOError:
        # If we can't read the file, return not found
        return return_response_not_found(request)
Example #7
0
def download_datafile(request, datafile_id):

    # todo handle missing file, general error
    datafile = Dataset_File.objects.get(pk=datafile_id)
    expid = datafile.dataset.experiment.id

    if has_datafile_access(request=request,
                           dataset_file_id=datafile.id):
        url = datafile.url
        if url.startswith('http://') or url.startswith('https://') \
            or url.startswith('ftp://'):
            return HttpResponseRedirect(datafile.url)

        elif datafile.protocol == 'tardis':
            file_path = path.join(settings.FILE_STORE_PATH,
                                  str(expid),
                                  str(datafile.dataset.id),
                                  datafile.filename)

            if not path.exists(file_path):
                file_path = path.join(settings.FILE_STORE_PATH,
                                      str(expid),
                                      datafile.filename)
                if not path.exists(file_path):
                    logger.error('exp %s: file not found: %s ' % (expid,
                                                                  datafile.url))
                    return return_response_not_found(request)

        elif datafile.protocol in ['', 'file']:
            file_path = datafile.url.partition('://')[2]
            if not path.exists(file_path):
                logger.error('exp %s: file not found: %s' % (expid,
                                                               datafile.url))
                return return_response_not_found(request)

        else:
            logger.error('exp %s: file protocol unknown: %s' % (expid,
                                                                datafile.url))
            return return_response_not_found(request)

    else:
        return return_response_error(request)

    try:
        wrapper = FileWrapper(file(file_path))
        response = HttpResponse(wrapper,
                                mimetype=datafile.get_mimetype())
        response['Content-Disposition'] = \
            'attachment; filename="%s"' % datafile.filename
        return response

    except IOError:
        logger.exception()
        return return_response_not_found(request)
Example #8
0
def download_datafile(request, datafile_id):

    # todo handle missing file, general error
    datafile = Dataset_File.objects.get(pk=datafile_id)

    if has_datafile_access(datafile.id, request.user):
        url = datafile.url

        if url.startswith("http://") or url.startswith("https://") or url.startswith("ftp://"):
            return HttpResponseRedirect(datafile.url)
        else:
            file_path = join(
                settings.FILE_STORE_PATH, str(datafile.dataset.experiment.id), datafile.url.partition("//")[2]
            )
            try:
                logger.debug(file_path)
                wrapper = FileWrapper(file(file_path))

                response = HttpResponse(wrapper, mimetype="application/octet-stream")
                response["Content-Disposition"] = 'attachment; filename="' + datafile.filename + '"'

                # import os
                # response['Content-Length'] = os.path.getsize(file_path)

                return response

            except IOError:
                return return_response_not_found(request)
    else:

        return return_response_error(request)
Example #9
0
def download_experiment(request, experiment_id, comptype):
    """
    takes string parameter "comptype" for compression method.
    Currently implemented: "zip" and "tar"
    """
    # Create the HttpResponse object with the appropriate headers.
    # TODO: handle no datafile, invalid filename, all http links
    # (tarfile count?)
    experiment = Experiment.objects.get(pk=experiment_id)

    if comptype == "tar":
        cmd = 'tar -C %s -c %s/' % (path.abspath(
            settings.FILE_STORE_PATH), str(experiment.id))
        # logger.info('TAR COMMAND: ' + cmd)
        response = HttpResponse(FileWrapper(
            subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout),
                                mimetype='application/x-tar')

        response['Content-Disposition'] = 'attachment; filename="experiment' \
            + str(experiment.id) + '-complete.tar"'
    elif comptype == "zip":
        cmd = 'cd %s; zip -r - %s' % (path.abspath(
            settings.FILE_STORE_PATH), str(experiment.id))
        # logger.info('TAR COMMAND: ' + cmd)
        response = HttpResponse(FileWrapper(
            subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout),
                                mimetype='application/zip')

        response['Content-Disposition'] = 'attachment; filename="experiment' \
            + str(experiment.id) + '-complete.zip"'
    else:
        return return_response_not_found(request)
    # response['Content-Length'] = fileSize + 5120
    return response
Example #10
0
def view_datafile(request, datafile_id):
    # Get datafile (and return 404 if absent)
    try:
        datafile = Dataset_File.objects.get(pk=datafile_id)
    except Dataset_File.DoesNotExist:
        return return_response_not_found(request)
    # Check users has access to datafile
    if not has_datafile_download_access(request=request,
                                        dataset_file_id=datafile.id):
        return return_response_error(request)
    # Get actual url for datafile
    file_url = _get_actual_url(datafile)
    mimetype = datafile.get_mimetype()
    if IMAGEMAGICK_AVAILABLE and mimetype in MIMETYPES_TO_VIEW_AS_PNG:
        with Image(file=urlopen(file_url)) as img:
            img.format = 'png'
            content = img.make_blob()
        # Should show up as a PNG file
        response = HttpResponse(content, mimetype='image/png')
        response['Content-Disposition'] = \
            'inline; filename="%s.png"' % datafile.filename
    else:
        response = _create_download_response(datafile, file_url,
                                             disposition='inline')
    return response
Example #11
0
def download_datafile(request, datafile_id):

    # todo handle missing file, general error
    datafile = Dataset_File.objects.get(pk=datafile_id)

    if has_datafile_access(request=request,
                           dataset_file_id=datafile.id):
        url = datafile.url

        if url.startswith('http://') or url.startswith('https://') \
            or url.startswith('ftp://'):
            return HttpResponseRedirect(datafile.url)
        else:
            file_path = datafile.get_absolute_filepath()

            try:
                wrapper = FileWrapper(file(file_path))
                response = HttpResponse(wrapper,
                                        mimetype=datafile.get_mimetype())
                response['Content-Disposition'] = \
                    'attachment; filename="%s"' % datafile.filename
                return response

            except IOError:
                return return_response_not_found(request)
    else:
        return return_response_error(request)
Example #12
0
def download_experiment(request, experiment_id, comptype):
    """
    takes string parameter "comptype" for compression method.
    Currently implemented: "zip" and "tar"
    """
    datafiles = Dataset_File.objects\
        .filter(dataset__experiments__id=experiment_id)

    if comptype == "tar":
        reader = StreamingFile(_write_tar_func(str(experiment_id), datafiles))

        response = HttpResponse(FileWrapper(reader),
                                mimetype='application/x-tar')
        response['Content-Disposition'] = 'attachment; filename="experiment' \
            + str(experiment_id) + '-complete.tar"'
    elif comptype == "zip":
        reader = StreamingFile(_write_zip_func(str(experiment_id), datafiles))
        response = HttpResponse(FileWrapper(reader),
                                mimetype='application/zip')

        response['Content-Disposition'] = 'attachment; filename="experiment' \
            + str(experiment_id) + '-complete.zip"'
    else:
        return return_response_not_found(request)
    # response['Content-Length'] = fileSize + 5120
    return response
Example #13
0
def download_experiment(request, experiment_id, comptype):
    """
    takes string parameter "comptype" for compression method.
    Currently implemented: "zip" and "tar"
    """
    # Create the HttpResponse object with the appropriate headers.
    # TODO: handle no datafile, invalid filename, all http links
    # (tarfile count?)
    experiment = Experiment.objects.get(pk=experiment_id)

    if comptype == "tar":
        cmd = "tar -C %s -c %s/" % (abspath(settings.FILE_STORE_PATH), str(experiment.id))
        # logger.info('TAR COMMAND: ' + cmd)
        response = HttpResponse(
            FileWrapper(subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout), mimetype="application/x-tar"
        )

        response["Content-Disposition"] = 'attachment; filename="experiment' + str(experiment.id) + '-complete.tar"'
    elif comptype == "zip":
        cmd = "cd %s; zip -r - %s" % (abspath(settings.FILE_STORE_PATH), str(experiment.id))
        # logger.info('TAR COMMAND: ' + cmd)
        response = HttpResponse(
            FileWrapper(subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout), mimetype="application/zip"
        )

        response["Content-Disposition"] = 'attachment; filename="experiment' + str(experiment.id) + '-complete.zip"'
    else:
        return return_response_not_found(request)
    # response['Content-Length'] = fileSize + 5120
    return response
Example #14
0
def download_experiment(request, experiment_id, comptype):
    """
    takes string parameter "comptype" for compression method.
    Currently implemented: "zip" and "tar"
    """
    datafiles = Dataset_File.objects\
        .filter(dataset__experiments__id=experiment_id)

    if comptype == "tar":
        reader = StreamingFile(_write_tar_func(str(experiment_id), datafiles))

        response = HttpResponse(FileWrapper(reader),
                                mimetype='application/x-tar')
        response['Content-Disposition'] = 'attachment; filename="experiment' \
            + str(experiment_id) + '-complete.tar"'
    elif comptype == "zip":
        reader = StreamingFile(_write_zip_func(str(experiment_id), datafiles))
        response = HttpResponse(FileWrapper(reader),
                                mimetype='application/zip')

        response['Content-Disposition'] = 'attachment; filename="experiment' \
            + str(experiment_id) + '-complete.zip"'
    else:
        return return_response_not_found(request)
    # response['Content-Length'] = fileSize + 5120
    return response
Example #15
0
def _create_download_response(request, datafile_id, disposition='attachment'):
    #import ipdb; ipdb.set_trace()
    # Get datafile (and return 404 if absent)
    try:
        datafile = Dataset_File.objects.get(pk=datafile_id)
    except Dataset_File.DoesNotExist:
        return return_response_not_found(request)
    # Check users has access to datafile
    if not has_datafile_download_access(request=request,
                                        dataset_file_id=datafile.id):
        return return_response_error(request)
    # Send an image that can be seen in the browser
    if disposition == 'inline' and datafile.is_image():
        from tardis.tardis_portal.iiif import download_image
        args = (request, datafile.id, 'full', 'full', '0', 'native')
        # Send unconverted image if web-compatible
        if datafile.get_mimetype() in ('image/gif', 'image/jpeg', 'image/png'):
            return download_image(*args)
        # Send converted image
        return download_image(*args, format='png')
    # Send local file
    try:
        # Get file object for datafile
        file_obj = datafile.get_file()
        if not file_obj:
            # If file path doesn't resolve, return not found
            return return_response_not_found(request)
        wrapper = FileWrapper(file_obj, blksize=65535)
        response = StreamingHttpResponse(wrapper,
                                         mimetype=datafile.get_mimetype())
        response['Content-Disposition'] = \
            '%s; filename="%s"' % (disposition, datafile.filename)
        return response
    except IOError:
        # If we can't read the file, return not found
        return return_response_not_found(request)
    except ValueError:  # raised when replica not verified TODO: custom excptn
        redirect = request.META.get('HTTP_REFERER',
                                    'http://%s/' %
                                    request.META.get('HTTP_HOST'))
        message = """The file you are trying to access has not yet been
                     verified. Verification is an automated background process.
                     Please try again later or contact the system
                     administrator if the issue persists."""
        message = ' '.join(message.split())  # removes spaces
        redirect = redirect + '#error:' + message
        return HttpResponseRedirect(redirect)
Example #16
0
def _create_download_response(request, datafile_id, disposition='attachment'):
    #import ipdb; ipdb.set_trace()
    # Get datafile (and return 404 if absent)
    try:
        datafile = Dataset_File.objects.get(pk=datafile_id)
    except Dataset_File.DoesNotExist:
        return return_response_not_found(request)
    # Check users has access to datafile
    if not has_datafile_download_access(request=request,
                                        dataset_file_id=datafile.id):
        return return_response_error(request)
    # Send an image that can be seen in the browser
    if disposition == 'inline' and datafile.is_image():
        from tardis.tardis_portal.iiif import download_image
        args = (request, datafile.id, 'full', 'full', '0', 'native')
        # Send unconverted image if web-compatible
        if datafile.get_mimetype() in ('image/gif', 'image/jpeg', 'image/png'):
            return download_image(*args)
        # Send converted image
        return download_image(*args, format='png')
    # Send local file
    try:
        # Get file object for datafile
        file_obj = datafile.get_file()
        if not file_obj:
            # If file path doesn't resolve, return not found
            return return_response_not_found(request)
        wrapper = FileWrapper(file_obj)
        response = HttpResponse(wrapper,
                                mimetype=datafile.get_mimetype())
        response['Content-Disposition'] = \
            '%s; filename="%s"' % (disposition, datafile.filename)
        return response
    except IOError:
        # If we can't read the file, return not found
        return return_response_not_found(request)
    except ValueError:  # raised when replica not verified TODO: custom excptn
        redirect = request.META.get('HTTP_REFERER',
                                    'http://%s/' %
                                    request.META.get('HTTP_HOST'))
        message = """The file you are trying to access has not yet been
                     verified. Verification is an automated background process.
                     Please try again later or contact the system
                     administrator if the issue persists."""
        message = ' '.join(message.split())  # removes spaces
        redirect = redirect + '#error:' + message
        return HttpResponseRedirect(redirect)
Example #17
0
def jobstatus(request, experiment_id):
    if not request.user.is_authenticated():
        return "Not logged in"
    try:
        #utils.update_job_status(experiment_id=experiment_id,
        #                        user_id=request.user.id)
        jobs = Job.objects.filter(experiment=Experiment.objects.get(
                pk=experiment_id))
        for job in jobs:
            job.updateStatus()
        datasets = jobs.values_list('dataset').distinct()
        logger.debug(repr(datasets))
        disparray = []
        for dataset in datasets:
            dataset = dataset[0]
            jobids = jobs.filter(dataset=dataset).values_list(
                'jobid').distinct()
            jobidarray = []
            for jobid in jobids:
                finished = True
                retrieved = True
                jobid = jobid[0]
                inttime = uuid.UUID(jobid).time
                submittime = datetime.datetime.fromtimestamp(
                    (inttime - 0x01b21dd213814000L)*100/1e9)
                thesejobs = jobs.filter(jobid=jobid)
                jobdataarray = []
                for job in thesejobs:
                    if job.jobstatus.strip() != "Finished":
                        finished = False
                    if job.jobstatus.strip() != "Retrieved":
                        retrieved = False
                    jobdata = {
                        'status': job.jobstatus,
                        'hpcjobid': job.hpcjobid,
                        'submittime': job.submittime,
                        }
                    jobdataarray.append(jobdata)
                jobiddict = {'jobid': jobid,
                             'joblist': jobdataarray,
                             'finished': finished,
                             'retrieved': retrieved,
                             'submittime': submittime.strftime(
                        "%d %b %Y, %H:%M:%S")}
                jobidarray.append(jobiddict)
            datasetdict = {'dataset': Dataset.objects.get(
                    pk=dataset).description,
                           'jobidlist': jobidarray}
            disparray.append(datasetdict)
            logger.debug(repr(disparray))
        c = Context({
                #'jobs': jobs,
                'disparray': disparray,
            })
    except Experiment.DoesNotExist:
        return return_response_not_found(request)

    return render_to_response('mrtardis/jobstatus.html', c)
Example #18
0
def download_datafile_ws(request):
    if 'url' in request.GET and len(request.GET['url']) > 0:
        url = urllib.unquote(request.GET['url'])
        raw_path = url.partition('//')[2]
        experiment_id = request.GET['experiment_id']
        datafile = Dataset_File.objects.filter(
            url__endswith=raw_path, dataset__experiment__id=experiment_id)[0]

        if has_datafile_access(request=request, dataset_file_id=datafile.id):

            file_path = datafile.get_absolute_filepath()

            try:
                wrapper = FileWrapper(file(file_path))

                response = HttpResponse(wrapper,
                                        mimetype=datafile.get_mimetype())
                response['Content-Disposition'] = \
                    'attachment; filename="%s"' % datafile.filename

                return response

            except IOError:
                try:
                    file_path = datafile.get_absolute_filepath_old()
                    wrapper = FileWrapper(file(file_path))

                    response = HttpResponse(wrapper,
                                            mimetype=datafile.get_mimetype())
                    response['Content-Disposition'] = \
                        'attachment; filename="%s"' % datafile.filename

                    return response
                except IOError:
                    return return_response_not_found(request)

        else:
            return return_response_not_found(request)

    else:
        return return_response_error(request)
Example #19
0
def download_datafile(request, datafile_id):
    # Get datafile (and return 404 if absent)
    try:
        datafile = Dataset_File.objects.get(pk=datafile_id)
    except Dataset_File.DoesNotExist:
        return return_response_not_found(request)
    # Check users has access to datafile
    if not has_datafile_download_access(request=request, dataset_file_id=datafile.id):
        return return_response_error(request)
    # Get actual url for datafile
    file_url = _get_actual_url(datafile)
    if not file_url:
        # If file path doesn't resolve, return not found
        return return_response_not_found(request)
    # Just redirect for remote files
    if not file_url.startswith('file://'):
        return HttpResponseRedirect(datafile.url)
    # Send local file
    try:
        return _create_download_response(datafile, file_url)
    except IOError:
        # If we can't read the file, return not found
        return return_response_not_found(request)
Example #20
0
def download_datafiles(request):

    # Create the HttpResponse object with the appropriate headers.
    # TODO: handle no datafile, invalid filename, all http links
    # (tarfile count?)
    expid = request.POST['expid']
    fileString = ''

    comptype = "zip"
    if 'comptype' in request.POST:
        comptype = request.POST['comptype']

    if 'datafile' in request.POST:
        if len(request.POST.getlist('datafile')) > 500:
            comptype = "tar"

    if 'dataset' in request.POST:
        comptype = "tar"  #todo quickfix, calc how many files

    # the following protocols can be handled by this module
    protocols = ['', 'file', 'tardis']
    known_protocols = len(protocols)
    if 'datafile' in request.POST or 'dataset' in request.POST:
        if (len(request.POST.getlist('datafile')) > 0 \
                or len(request.POST.getlist('dataset'))) > 0:

            datasets = request.POST.getlist('dataset')
            datafiles = request.POST.getlist('datafile')

            for dsid in datasets:
                for datafile in Dataset_File.objects.filter(dataset=dsid):
                    if has_datafile_access(request=request,
                                           dataset_file_id=datafile.id):
                        p = datafile.protocol
                        if not p in protocols:
                            protocols += [p]
                        absolute_filename = datafile.url.partition('//')[2]
                        if (datafile.url.partition('//')[0] == 'tardis:'):
                            #temp fix for old data
                            filepath = '%s/%s/%s' %\
                            (expid, str(datafile.dataset.id),
                                absolute_filename)

                            print filepath + "######"

                            try:
                                wrapper = FileWrapper(file(
                                    datafile.get_absolute_filepath()))\
                                #exists test. os.exists broken

                            except IOError:
                                print "OLD FILE DETECTED"
                                filepath = '%s/%s' % (expid, absolute_filename)

                            fileString += ('\"' + filepath + '\" ')
                            print fileString
                        else:
                            fileString += '\"%s/%s\" ' %\
                            (expid, absolute_filename)

            for dfid in datafiles:
                datafile = Dataset_File.objects.get(pk=dfid)
                if datafile.dataset.id in datasets:
                    continue
                if has_datafile_access(request=request,
                                       dataset_file_id=datafile.id):
                    p = datafile.protocol
                    if not p in protocols:
                        protocols += [p]
                    absolute_filename = datafile.url.partition('//')[2]
                    if (datafile.url.partition('//')[0] == 'tardis:'):
                        #temp fix for old data
                        filepath = '\"%s/%s/%s\" ' %\
                        (expid, str(datafile.dataset.id),
                            absolute_filename)

                        print filepath + "######"

                        try:
                            wrapper = FileWrapper(file(
                                datafile.get_absolute_filepath()))\
                            #exists test. os.exists broken

                        except IOError:
                            print "OLD FILE DETECTED"
                            filepath = '\"%s/%s\" ' %\
                                       (expid, absolute_filename)

                        fileString += filepath
                        print fileString
                    else:
                        fileString += '\"%s/%s\" ' % (expid, absolute_filename)
        else:
            return return_response_not_found(request)

    elif 'url' in request.POST:
        if not len(request.POST.getlist('url')) == 0:
            comptype = "tar"  #todo quickfix for zip error
            fileString = ""
            for url in request.POST.getlist('url'):
                url = urllib.unquote(url)
                raw_path = url.partition('//')[2]
                experiment_id = request.POST['expid']
                datafile = Dataset_File.objects.filter(
                    url__endswith=raw_path,
                    dataset__experiment__id=experiment_id)[0]
                if has_datafile_access(request=request,
                                       dataset_file_id=datafile.id):
                    p = datafile.protocol
                    if not p in protocols:
                        protocols += [p]
                    absolute_filename = datafile.url.partition('//')[2]
                    if (datafile.url.partition('//')[0] == 'tardis:'):
                        # expects tardis: formatted stuff
                        # to not include dataset id

                        #temp fix for old data
                        filepath = '\"%s/%s/%s\" ' %\
                            (expid, str(datafile.dataset.id),
                            absolute_filename)

                        print filepath + "######"

                        try:
                            wrapper = FileWrapper(file(
                                datafile.get_absolute_filepath()))\
                            #exists test. os.exists broken

                        except IOError:
                            print "OLD FILE DETECTED"
                            filepath = '\"%s/%s\" ' %\
                                       (expid, absolute_filename)

                        fileString += ('\"' + filepath + '\" ')
                        print fileString
                    else:
                        fileString += '\"%s/%s\" ' % (expid, absolute_filename)
        else:
            return return_response_not_found(request)
    else:
        return return_response_not_found(request)

    # more than one external download location?
    if len(protocols) > known_protocols + 2:
        response = HttpResponseNotFound()
        response.write('<p>Different locations selected!</p>\n')
        response.write('Please limit your selection and try again.\n')
        return response

    # redirect request if another (external) download protocol was found
    elif len(protocols) == known_protocols + 1:
        from django.core.urlresolvers import reverse, resolve
        try:
            for module in settings.DOWNLOAD_PROVIDERS:
                if module[0] == protocols[3]:
                    url = reverse('%s.download_datafiles' % module[1])
                    view, args, kwargs = resolve(url)
                    kwargs['request'] = request
                    return view(*args, **kwargs)
        except:
            return return_response_not_found(request)

    else:
        # tarfile class doesn't work on large files being added and
        # streamed on the fly, so going command-line-o
        if not fileString:
            return return_response_error(request)

        if comptype == "tar":
            cmd = 'tar -C %s -c %s' % (settings.FILE_STORE_PATH, fileString)

            # logger.info(cmd)
            response = \
                HttpResponse(FileWrapper(subprocess.Popen(
                                                    cmd,
                                                    stdout=subprocess.PIPE,
                                                    shell=True).stdout),
                             mimetype='application/x-tar')
            response['Content-Disposition'] = \
                    'attachment; filename="experiment%s-selection.tar"' % expid
            return response
        else:
            cmd = 'cd %s; zip -r - %s' % (settings.FILE_STORE_PATH, fileString)
            # logger.info(cmd)
            response = \
                HttpResponse(FileWrapper(subprocess.Popen(
                                                    cmd,
                                                    stdout=subprocess.PIPE,
                                                    shell=True).stdout),
                             mimetype='application/zip')
            response['Content-Disposition'] = \
                    'attachment; filename="experiment%s-selection.zip"' % expid
            return response
Example #21
0
def download_datafiles(request):

    # Create the HttpResponse object with the appropriate headers.
    # TODO: handle no datafile, invalid filename, all http links
    # (tarfile count?)

    comptype = "zip"
    if 'comptype' in request.POST:
        comptype = request.POST['comptype']

    if 'datafile' in request.POST or 'dataset' in request.POST:
        if (len(request.POST.getlist('datafile')) > 0 \
                or len(request.POST.getlist('dataset'))) > 0:

            datasets = request.POST.getlist('dataset')
            datafiles = request.POST.getlist('datafile')

            # Generator to produce datafiles from dataset id
            def get_dataset_datafiles(dsid):
                for datafile in Dataset_File.objects.filter(dataset=dsid):
                    if has_datafile_download_access(
                            request=request, dataset_file_id=datafile.id):
                        yield datafile

            # Generator to produce datafile from datafile id
            def get_datafile(dfid):
                datafile = Dataset_File.objects.get(pk=dfid)
                if has_datafile_download_access(request=request,
                                                dataset_file_id=datafile.id):
                    yield datafile

            # Take chained generators and turn them into a set of datafiles
            df_set = set(
                chain(
                    chain.from_iterable(map(get_dataset_datafiles, datasets)),
                    chain.from_iterable(map(get_datafile, datafiles))))
        else:
            return return_response_not_found(request)

    elif 'url' in request.POST:
        if not len(request.POST.getlist('url')) == 0:
            return return_response_not_found(request)

        for url in request.POST.getlist('url'):
            url = urllib.unquote(url)
            raw_path = url.partition('//')[2]
            experiment_id = request.POST['expid']
            datafile = Dataset_File.objects.filter(
                url__endswith=raw_path,
                dataset__experiment__id=experiment_id)[0]
            if has_datafile_download_access(request=request,
                                            dataset_file_id=datafile.id):
                df_set = set([datafile])
    else:
        return return_response_not_found(request)

    logger.info('Files for archive command: %s' % df_set)

    if len(df_set) == 0:
        return return_response_error(request)

    # Handle missing experiment ID - only need it for naming
    try:
        expid = request.POST['expid']
    except KeyError:
        expid = iter(df_set).next().dataset.get_first_experiment().id

    if comptype == "tar":
        reader = StreamingFile(_write_tar_func('datasets', df_set))

        # logger.info(cmd)
        response = \
            HttpResponse(FileWrapper(reader),
                         mimetype='application/x-tar')
        response['Content-Disposition'] = \
                'attachment; filename="experiment%s-selection.tar"' % expid
        return response
    else:
        reader = StreamingFile(_write_zip_func('datasets', df_set))

        # logger.info(cmd)
        response = \
            HttpResponse(FileWrapper(reader),
                         mimetype='application/zip')
        response['Content-Disposition'] = \
                'attachment; filename="experiment%s-selection.zip"' % expid
        return response
Example #22
0
def download_datafiles(request):

    # Create the HttpResponse object with the appropriate headers.
    # TODO: handle no datafile, invalid filename, all http links
    # (tarfile count?)
    expid = request.POST["expid"]
    protocols = []
    fileString = ""
    fileSize = 0

    if "datafile" or "dataset" in request.POST:

        if (len(request.POST.getlist("datafile")) > 0 or len(request.POST.getlist("dataset"))) > 0:

            datasets = request.POST.getlist("dataset")
            datafiles = request.POST.getlist("datafile")

            for dsid in datasets:
                for datafile in Dataset_File.objects.filter(dataset=dsid):
                    if has_datafile_access(datafile.id, request.user):
                        p = datafile.protocol
                        if not p in protocols:
                            protocols += [p]
                        if datafile.url.startswith("file://"):
                            absolute_filename = datafile.url.partition("//")[2]
                            fileString += expid + "/" + absolute_filename + " "
                            fileSize += long(datafile.size)

            for dfid in datafiles:
                datafile = Dataset_File.objects.get(pk=dfid)
                if datafile.dataset.id in datasets:
                    continue
                if has_datafile_access(dfid, request.user):
                    p = datafile.protocol
                    if not p in protocols:
                        protocols += [p]
                    if datafile.url.startswith("file://"):
                        absolute_filename = datafile.url.partition("//")[2]
                        fileString += expid + "/" + absolute_filename + " "
                        fileSize += long(datafile.size)

        else:
            return return_response_not_found(request)

    elif "url" in request.POST:

        if not len(request.POST.getlist("url")) == 0:
            for url in request.POST.getlist("url"):
                datafile = Dataset_File.objects.get(
                    url=urllib.unquote(url), dataset__experiment__id=request.POST["expid"]
                )
                if has_datafile_access(datafile.id, request.user):
                    p = datafile.protocol
                    if not p in protocols:
                        protocols += [p]
                    if datafile.url.startswith("file://"):
                        absolute_filename = datafile.url.partition("//")[2]
                        fileString += expid + "/" + absolute_filename + " "
                        fileSize += long(datafile.size)

        else:
            return return_response_not_found(request)
    else:
        return return_response_not_found(request)

    # more than one download location?
    if len(protocols) > 1:
        response = HttpResponseNotFound()
        response.write("<p>Different locations selected!</p>\n")
        response.write("Please limit your selection and try again.\n")
        return response

    # redirect request if download protocol found
    if protocols[0] != "":
        from django.core.urlresolvers import resolve

        view, args, kwargs = resolve("/%s%s" % (protocols[0], request.path))
        kwargs["request"] = request
        return view(*args, **kwargs)

    else:
        # tarfile class doesn't work on large files being added and
        # streamed on the fly, so going command-line-o
        cmd = "tar -C %s -c %s" % (settings.FILE_STORE_PATH, fileString)

        response = HttpResponse(
            FileWrapper(subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout), mimetype="application/x-tar"
        )
        response["Content-Disposition"] = "attachment; filename=experiment%s.tar" % expid
        response["Content-Length"] = fileSize + 5120
        return response
Example #23
0
def download_datafiles(request):

    # Create the HttpResponse object with the appropriate headers.
    # TODO: handle no datafile, invalid filename, all http links
    # (tarfile count?)
    expid = request.POST['expid']
    fileString = ''

    comptype = "zip"
    if 'comptype' in request.POST:
        comptype = request.POST['comptype']

    if 'datafile' in request.POST:
        if len(request.POST.getlist('datafile')) > 500:
            comptype = "tar"

    if 'dataset' in request.POST:
        comptype = "tar" #todo quickfix, calc how many files

    # the following protocols can be handled by this module
    protocols = ['', 'file', 'tardis']
    known_protocols = len(protocols)
    if 'datafile' in request.POST or 'dataset' in request.POST:
        if (len(request.POST.getlist('datafile')) > 0 \
                or len(request.POST.getlist('dataset'))) > 0:

            datasets = request.POST.getlist('dataset')
            datafiles = request.POST.getlist('datafile')

            for dsid in datasets:
                for datafile in Dataset_File.objects.filter(dataset=dsid):
                    if has_datafile_access(request=request,
                                            dataset_file_id=datafile.id):
                        p = datafile.protocol
                        if not p in protocols:
                            protocols += [p]
                        absolute_filename = datafile.url.partition('//')[2]
                        if(datafile.url.partition('//')[0] == 'tardis:'):
                            #temp fix for old data
                            filepath = '%s/%s/%s' %\
                            (expid, str(datafile.dataset.id),
                                absolute_filename)

                            print filepath + "######"

                            try:
                                wrapper = FileWrapper(file(
                                    datafile.get_absolute_filepath()))\
                                #exists test. os.exists broken
                            except IOError:
                                print "OLD FILE DETECTED"
                                filepath = '%s/%s' % (expid, absolute_filename)

                            fileString += ('\"' + filepath + '\" ')
                            print fileString
                        else:
                            fileString += '\"%s/%s\" ' %\
                            (expid, absolute_filename)


            for dfid in datafiles:
                datafile = Dataset_File.objects.get(pk=dfid)
                if datafile.dataset.id in datasets:
                    continue
                if has_datafile_access(request=request,
                        dataset_file_id=datafile.id):
                    p = datafile.protocol
                    if not p in protocols:
                        protocols += [p]
                    absolute_filename = datafile.url.partition('//')[2]
                    if(datafile.url.partition('//')[0] == 'tardis:'):
                        #temp fix for old data
                        filepath = '\"%s/%s/%s\" ' %\
                        (expid, str(datafile.dataset.id),
                            absolute_filename)

                        print filepath + "######"

                        try:
                            wrapper = FileWrapper(file(
                                datafile.get_absolute_filepath()))\
                            #exists test. os.exists broken
                        except IOError:
                            print "OLD FILE DETECTED"
                            filepath = '\"%s/%s\" ' %\
                                       (expid, absolute_filename)

                        fileString += filepath
                        print fileString
                    else:
                        fileString += '\"%s/%s\" ' % (expid, absolute_filename)
        else:
            return return_response_not_found(request)

    elif 'url' in request.POST:
        if not len(request.POST.getlist('url')) == 0:
            comptype = "tar" #todo quickfix for zip error
            fileString = ""
            for url in request.POST.getlist('url'):
                url = urllib.unquote(url)
                raw_path = url.partition('//')[2]
                experiment_id = request.POST['expid']
                datafile = Dataset_File.objects.filter(url__endswith=raw_path,
                    dataset__experiment__id=experiment_id)[0]
                if has_datafile_access(request=request,
                                       dataset_file_id=datafile.id):
                    p = datafile.protocol
                    if not p in protocols:
                        protocols += [p]
                    absolute_filename = datafile.url.partition('//')[2]
                    if(datafile.url.partition('//')[0] == 'tardis:'):
                        # expects tardis: formatted stuff
                        # to not include dataset id

                        #temp fix for old data
                        filepath = '\"%s/%s/%s\" ' %\
                            (expid, str(datafile.dataset.id),
                            absolute_filename)

                        print filepath + "######"

                        try:
                            wrapper = FileWrapper(file(
                                datafile.get_absolute_filepath()))\
                            #exists test. os.exists broken
                        except IOError:
                            print "OLD FILE DETECTED"
                            filepath = '\"%s/%s\" ' %\
                                       (expid, absolute_filename)

                        fileString += ('\"' + filepath + '\" ')
                        print fileString
                    else:
                        fileString += '\"%s/%s\" ' % (expid, absolute_filename)
        else:
            return return_response_not_found(request)
    else:
        return return_response_not_found(request)

    # more than one external download location?
    if len(protocols) > known_protocols + 2:
        response = HttpResponseNotFound()
        response.write('<p>Different locations selected!</p>\n')
        response.write('Please limit your selection and try again.\n')
        return response

    # redirect request if another (external) download protocol was found
    elif len(protocols) == known_protocols + 1:
        from django.core.urlresolvers import reverse, resolve
        try:
            for module in settings.DOWNLOAD_PROVIDERS:
                if module[0] == protocols[3]:
                    url = reverse('%s.download_datafiles' % module[1])
                    view, args, kwargs = resolve(url)
                    kwargs['request'] = request
                    return view(*args, **kwargs)
        except:
            return return_response_not_found(request)

    else:
        # tarfile class doesn't work on large files being added and
        # streamed on the fly, so going command-line-o
        if not fileString:
            return return_response_error(request)

        if comptype == "tar":
            cmd = 'tar -C %s -c %s' % (settings.FILE_STORE_PATH,
                                       fileString)

            # logger.info(cmd)
            response = \
                HttpResponse(FileWrapper(subprocess.Popen(
                                                    cmd,
                                                    stdout=subprocess.PIPE,
                                                    stderr=open(devnull, 'w'),
                                                    shell=True).stdout),
                             mimetype='application/x-tar')
            response['Content-Disposition'] = \
                    'attachment; filename="experiment%s-selection.tar"' % expid
            return response
        else:
            cmd = 'cd %s; zip -r - %s' % (settings.FILE_STORE_PATH,
                                       fileString)
            # logger.info(cmd)
            response = \
                HttpResponse(FileWrapper(subprocess.Popen(
                                                    cmd,
                                                    stdout=subprocess.PIPE,
                                                    stderr=open(devnull, 'w'),
                                                    shell=True).stdout),
                             mimetype='application/zip')
            response['Content-Disposition'] = \
                    'attachment; filename="experiment%s-selection.zip"' % expid
            return response
Example #24
0
def download_datafiles(request):

    # Create the HttpResponse object with the appropriate headers.
    # TODO: handle no datafile, invalid filename, all http links
    # (tarfile count?)
    expid = request.POST["expid"]
    fileString = ""
    fileSize = 0

    comptype = "zip"
    if "comtype" in request.POST:
        comptype = request.POST["comptype"]

    # the following protocols can be handled by this module
    protocols = ["", "file", "tardis"]
    known_protocols = len(protocols)

    if "datafile" or "dataset" in request.POST:
        if (len(request.POST.getlist("datafile")) > 0 or len(request.POST.getlist("dataset"))) > 0:

            datasets = request.POST.getlist("dataset")
            datafiles = request.POST.getlist("datafile")

            for dsid in datasets:
                for datafile in Dataset_File.objects.filter(dataset=dsid):
                    if has_datafile_access(request=request, dataset_file_id=datafile.id):
                        p = datafile.protocol
                        if not p in protocols:
                            protocols += [p]
                        absolute_filename = datafile.url.partition("//")[2]
                        if datafile.url.partition("//")[0] == "tardis:":
                            fileString += "%s/%s/%s " % (expid, str(datafile.dataset.id), absolute_filename)
                        else:
                            fileString += "%s/%s " % (expid, absolute_filename)
                        fileSize += long(datafile.size)

            for dfid in datafiles:
                datafile = Dataset_File.objects.get(pk=dfid)
                if datafile.dataset.id in datasets:
                    continue
                if has_datafile_access(request=request, dataset_file_id=datafile.id):
                    p = datafile.protocol
                    if not p in protocols:
                        protocols += [p]
                    absolute_filename = datafile.url.partition("//")[2]
                    if datafile.url.partition("//")[0] == "tardis:":
                        fileString += "%s/%s/%s " % (expid, str(datafile.dataset.id), absolute_filename)
                    else:
                        fileString += "%s/%s " % (expid, absolute_filename)
                    fileSize += long(datafile.size)
        else:
            return return_response_not_found(request)

    elif "url" in request.POST:
        if not len(request.POST.getlist("url")) == 0:
            fileString = ""
            fileSize = 0
            for url in request.POST.getlist("url"):
                url = urllib.unquote(url)
                raw_path = url.partition("//")[2]
                experiment_id = request.POST["expid"]
                datafile = Dataset_File.objects.filter(url__endswith=raw_path, dataset__experiment__id=experiment_id)[0]
                if has_datafile_access(request=request, dataset_file_id=datafile.id):
                    p = datafile.protocol
                    if not p in protocols:
                        protocols += [p]
                    absolute_filename = datafile.url.partition("//")[2]
                    if datafile.url.partition("//")[0] == "tardis:":
                        # expects tardis: formatted stuff to not include dataset id
                        fileString += "%s/%s/%s " % (expid, str(datafile.dataset.id), absolute_filename)
                    else:
                        fileString += "%s/%s " % (expid, absolute_filename)
                    fileSize += long(datafile.size)
        else:
            return return_response_not_found(request)
    else:
        return return_response_not_found(request)

    # more than one external download location?
    if len(protocols) > known_protocols + 2:
        response = HttpResponseNotFound()
        response.write("<p>Different locations selected!</p>\n")
        response.write("Please limit your selection and try again.\n")
        return response

    # redirect request if another (external) download protocol was found
    elif len(protocols) == known_protocols + 1:
        from django.core.urlresolvers import reverse, resolve

        try:
            for module in settings.DOWNLOAD_PROVIDERS:
                if module[0] == protocols[3]:
                    url = reverse("%s.download_datafiles" % module[1])
                    view, args, kwargs = resolve(url)
                    kwargs["request"] = request
                    return view(*args, **kwargs)
        except:
            return return_response_not_found(request)

    else:
        # tarfile class doesn't work on large files being added and
        # streamed on the fly, so going command-line-o
        if not fileString:
            return return_response_error(request)

        if comptype == "tar":
            cmd = "tar -C %s -c %s" % (settings.FILE_STORE_PATH, fileString)

            # logger.info(cmd)
            response = HttpResponse(
                FileWrapper(subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout),
                mimetype="application/x-tar",
            )
            response["Content-Disposition"] = 'attachment; filename="experiment%s-selection.tar"' % expid
            response["Content-Length"] = fileSize + 5120
            return response
        else:
            cmd = "cd %s; zip -r - %s" % (settings.FILE_STORE_PATH, fileString)

            # logger.info(cmd)
            response = HttpResponse(
                FileWrapper(subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout),
                mimetype="application/zip",
            )
            response["Content-Disposition"] = 'attachment; filename="experiment%s-selection.zip"' % expid
            response["Content-Length"] = fileSize + 5120
            return response
Example #25
0
def download_datafiles(request):

    # Create the HttpResponse object with the appropriate headers.
    # TODO: handle no datafile, invalid filename, all http links
    # (tarfile count?)
    expid = request.POST['expid']
    protocols = []
    fileString = ''
    fileSize = 0

    # the following protocols can be handled by this module
    protocols = ['', 'file', 'tardis']

    if 'datafile' or 'dataset' in request.POST:

        if (len(request.POST.getlist('datafile')) > 0 \
                or len(request.POST.getlist('dataset'))) > 0:

            datasets = request.POST.getlist('dataset')
            datafiles = request.POST.getlist('datafile')

            for dsid in datasets:
                for datafile in Dataset_File.objects.filter(dataset=dsid):
                    if has_datafile_access(request=request,
                                           dataset_file_id=datafile.id):
                        p = datafile.protocol
                        if not p in protocols:
                            protocols += [p]

                        absolute_filename = datafile.url.partition('//')[2]
                        fileString += '%s/%s ' % (expid, absolute_filename)
                        fileSize += long(datafile.size)

            for dfid in datafiles:
                datafile = Dataset_File.objects.get(pk=dfid)
                if datafile.dataset.id in datasets:
                    continue
                if has_datafile_access(request=request,
                                        dataset_file_id=datafile.id):
                    p = datafile.protocol
                    if not p in protocols:
                        protocols += [p]
                    absolute_filename = datafile.url.partition('//')[2]
                    fileString += '%s/%s ' % (expid, absolute_filename)
                    fileSize += long(datafile.size)

        else:
            return return_response_not_found(request)

    # TODO: check if we really still need this method
    elif 'url' in request.POST:

        if not len(request.POST.getlist('url')) == 0:
            for url in request.POST.getlist('url'):
                datafile = \
                    Dataset_File.objects.get(url=urllib.unquote(url),
                        dataset__experiment__id=request.POST['expid'])
                if has_datafile_access(request=request,
                                       dataset_file_id=datafile.id):
                    p = datafile.protocol
                    if not p in protocols:
                        protocols += [p]
                    absolute_filename = datafile.url.partition('//')[2]
                    fileString += '%s/%s ' % (expid, absolute_filename)
                    fileSize += long(datafile.size)

        else:
            return return_response_not_found(request)
    else:
        return return_response_not_found(request)

    # more than one external download location?
    if len(protocols) > 4:
        response = HttpResponseNotFound()
        response.write('<p>Different locations selected!</p>\n')
        response.write('Please limit your selection and try again.\n')
        return response

    # redirect request if another (external) download protocol was found
    elif len(protocols) == 4:
        from django.core.urlresolvers import resolve
        view, args, kwargs = resolve('/%s%s' % (protocols[4],
                                                request.path))
        kwargs['request'] = request
        return view(*args, **kwargs)

    else:
        # tarfile class doesn't work on large files being added and
        # streamed on the fly, so going command-line-o
        if not fileString:
            return return_response_error(request)

        cmd = 'tar -C %s -c %s' % (settings.FILE_STORE_PATH,
                                   fileString)

        # logger.info(cmd)
        response = \
            HttpResponse(FileWrapper(subprocess.Popen(cmd,
                                                      stdout=subprocess.PIPE,
                                                      shell=True).stdout),
                         mimetype='application/x-tar')
        response['Content-Disposition'] = \
                'attachment; filename="experiment%s.tar"' % expid
        response['Content-Length'] = fileSize + 5120
        return response
Example #26
0
def download_datafiles(request):

    # Create the HttpResponse object with the appropriate headers.
    # TODO: handle no datafile, invalid filename, all http links
    # (tarfile count?)

    comptype = "zip"
    if 'comptype' in request.POST:
        comptype = request.POST['comptype']

    if 'datafile' in request.POST or 'dataset' in request.POST:
        if (len(request.POST.getlist('datafile')) > 0 \
                or len(request.POST.getlist('dataset'))) > 0:

            datasets = request.POST.getlist('dataset')
            datafiles = request.POST.getlist('datafile')

            # Generator to produce datafiles from dataset id
            def get_dataset_datafiles(dsid):
                for datafile in Dataset_File.objects.filter(dataset=dsid):
                    if has_datafile_download_access(request=request,
                                                    dataset_file_id=datafile.id):
                        yield datafile


            # Generator to produce datafile from datafile id
            def get_datafile(dfid):
                datafile = Dataset_File.objects.get(pk=dfid)
                if has_datafile_download_access(request=request,
                                                dataset_file_id=datafile.id):
                    yield datafile

            # Take chained generators and turn them into a set of datafiles
            df_set = set(chain(chain.from_iterable(map(get_dataset_datafiles,
                                                       datasets)),
                               chain.from_iterable(map(get_datafile,
                                                       datafiles))))
        else:
            return return_response_not_found(request)

    elif 'url' in request.POST:
        if not len(request.POST.getlist('url')) == 0:
            return return_response_not_found(request)

        for url in request.POST.getlist('url'):
            url = urllib.unquote(url)
            raw_path = url.partition('//')[2]
            experiment_id = request.POST['expid']
            datafile = Dataset_File.objects.filter(url__endswith=raw_path,
                dataset__experiment__id=experiment_id)[0]
            if has_datafile_download_access(request=request,
                                            dataset_file_id=datafile.id):
                df_set = set([datafile])
    else:
        return return_response_not_found(request)

    logger.info('Files for archive command: %s' % df_set)

    if len(df_set) == 0:
        return return_response_error(request)

    # Handle missing experiment ID - only need it for naming
    try:
        expid = request.POST['expid']
    except KeyError:
        expid = iter(df_set).next().dataset.get_first_experiment().id

    if comptype == "tar":
        reader = StreamingFile(_write_tar_func('datasets', df_set))

        # logger.info(cmd)
        response = \
            HttpResponse(FileWrapper(reader),
                         mimetype='application/x-tar')
        response['Content-Disposition'] = \
                'attachment; filename="experiment%s-selection.tar"' % expid
        return response
    else:
        reader = StreamingFile(_write_zip_func('datasets', df_set))

        # logger.info(cmd)
        response = \
            HttpResponse(FileWrapper(reader),
                         mimetype='application/zip')
        response['Content-Disposition'] = \
                'attachment; filename="experiment%s-selection.zip"' % expid
        return response
Example #27
0
def download_datafile(request, datafile_id):

    # todo handle missing file, general error
    datafile = Dataset_File.objects.get(pk=datafile_id)
    expid = datafile.dataset.experiment.id

    if has_datafile_access(request=request, dataset_file_id=datafile.id):
        url = datafile.url
        if url.startswith('http://') or url.startswith('https://') \
            or url.startswith('ftp://'):
            return HttpResponseRedirect(datafile.url)

        elif datafile.protocol == 'tardis' or \
            datafile.url.startswith('tardis'):

            raw_path = url.partition('//')[2]
            file_path = path.join(settings.FILE_STORE_PATH, str(expid),
                                  str(datafile.dataset.id), raw_path)
            try:

                wrapper = FileWrapper(file(file_path))
                response = HttpResponse(wrapper,
                                        mimetype=datafile.get_mimetype())
                response['Content-Disposition'] = \
                    'attachment; filename="%s"' % datafile.filename
                return response

            except IOError:
                try:
                    file_path = path.join(settings.FILE_STORE_PATH, str(expid),
                                          raw_path)
                    print file_path
                    wrapper = FileWrapper(file(file_path))
                    response = HttpResponse(wrapper,
                                            mimetype=datafile.get_mimetype())
                    response['Content-Disposition'] = \
                        'attachment; filename="%s"' % datafile.filename
                    return response
                except IOError:
                    return return_response_not_found(request)

        elif datafile.protocol in ['', 'file']:
            file_path = datafile.url.partition('://')[2]
            if not path.exists(file_path):

                return return_response_not_found(request)

        else:
            #            logger.error('exp %s: file protocol unknown: %s' % (expid,
            #                                                                datafile.url))
            return return_response_not_found(request)

    else:
        return return_response_error(request)

    try:
        wrapper = FileWrapper(file(file_path))
        response = HttpResponse(wrapper, mimetype=datafile.get_mimetype())
        response['Content-Disposition'] = \
            'attachment; filename="%s"' % datafile.filename
        return response

    except IOError:
        logger.exception()
        return return_response_not_found(request)