def respuesta(sock):
try:
OUT = sock.makefile()
http_command = OUT.readline()
headers = []
command = http_command.split()
#print "Commando:", command
for line in OUT:
if line.strip() == "": break
headers.append(line)
if command[0] == "GET":
if command[1] == "/":
filename = document_root + "/index.html"
else:
filename = document_root + command[1]
try:
FILE = open(filename, "rb")
#print "Sending", filename
OUT.write("HTTP/1.0 200 Va be\r\n")
last_modified = os.stat(filename)[ST_MTIME]
OUT.write("Last-Modified: %s \r\n" % last_modified)
content_type = mimetypes.guess_type(filename)
OUT.write("Content-Type: %s\r\n" % content_type[0])
size = os.stat(filename)[ST_SIZE]
OUT.write("Content-Length: %s\r\n\r\n" % size)
OUT.flush()
sendfile.sendfile(OUT.fileno(), FILE.fileno(), 0, size)
except IOError:
OUT.write("HTTP 404 Fichero no existe\r\n")
print "Error con", filename
except TypeError,e: #Exception, e:
print "Error en la conexión", e
def test_flags(self):
try:
sendfile.sendfile(self.sockno, self.fileno, 0, BUFFER_LEN, flags=sendfile.SF_NODISKIO)
except OSError:
err = sys.exc_info()[1]
if err.errno not in (errno.EBUSY, errno.EAGAIN):
raise
def _download(self, image_href, data=None, method='data'):
"""Calls out to Glance for data and writes data.
:param image_href: The opaque image identifier.
:param data: (Optional) File object to write data to.
"""
image_id = service_utils.parse_image_id(image_href)
if 'file' in CONF.glance.allowed_direct_url_schemes:
location = self._get_location(image_id)
url = urlparse.urlparse(location)
if url.scheme == "file":
with open(url.path, "r") as f:
filesize = os.path.getsize(f.name)
sendfile.sendfile(data.fileno(), f.fileno(), 0, filesize)
return
image_chunks = self.call(method, image_id)
# NOTE(dtantsur): when using Glance V2, image_chunks is a wrapper
# around real data, so we have to check the wrapped data for None.
if image_chunks.wrapped is None:
raise exception.ImageDownloadFailed(
image_href=image_href, reason=_('image contains no data.'))
if data is None:
return image_chunks
else:
for chunk in image_chunks:
data.write(chunk)
def respuesta(sock):
try:
OUT = sock.makefile()
http_command = OUT.readline()
headers = []
command = http_command.split()
# print "Commando:", command
for line in OUT:
if line.strip() == "":
break
headers.append(line)
if command[0] == "GET":
if command[1] == "/":
filename = document_root + "/index.html"
else:
filename = document_root + command[1]
try:
FILE = open(filename, "rb")
# print "Sending", filename
m = mim.guess_type(filename)
size = os.stat(filename)[ST_SIZE]
mod = os.stat(filename)[ST_MTIME]
t = time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime(mod))
OUT.write("HTTP/1.0 200 OK\r\n")
OUT.write("Last-Modified: %s\r\n" % t)
OUT.write("Content-Type: %s\r\n" % m[0])
OUT.write("Content-Length: %d\r\n\r\n" % size)
OUT.flush()
sendfile.sendfile(OUT.fileno(), FILE.fileno(), 0, size)
except IOError:
OUT.write("HTTP 404 Fichero no existe\r\n")
# print "Error con", filename
except Exception, e:
print "Error en la conexión", e
def _download(self, image_href, data=None, method='data'):
"""Calls out to Glance for data and writes data.
:param image_id: The opaque image identifier.
:param data: (Optional) File object to write data to.
"""
image_id = service_utils.parse_image_ref(image_href)[0]
if (self.version == 2 and
'file' in CONF.glance.allowed_direct_url_schemes):
location = self._get_location(image_id)
url = urlparse.urlparse(location)
if url.scheme == "file":
with open(url.path, "r") as f:
filesize = os.path.getsize(f.name)
sendfile.sendfile(data.fileno(), f.fileno(), 0, filesize)
return
image_chunks = self.call(method, image_id)
if data is None:
return image_chunks
else:
for chunk in image_chunks:
data.write(chunk)
def stream(request, path):
basedir = Directory.get_basedir().absolute_path()
file_path = str(basedir / path)
mime_type = 'audio/ogg'
if(file_path.lower().endswith(('mp3', 'ogg', 'flac'))):
return sendfile(
request,
file_path,
attachment=False,
mimetype=mime_type,
)
file_path_without_ext = os.path.splitext(file_path)[0]
new_file_path = '/tmp' + file_path_without_ext + '.ogg'
if not os.path.isfile(new_file_path):
transcode_audio(file_path, new_file_path)
return StreamingHttpResponse(
stream_audio(file_path),
content_type=mime_type
)
return sendfile(
request,
new_file_path,
root_url='/tmp' + settings.SENDFILE_URL,
root_directory='/tmp' + settings.SENDFILE_ROOT,
attachment=False,
mimetype=mime_type,
)
def download(self, image_href, image_file):
"""Downloads image to specified location.
:param image_href: Image reference.
:param image_file: File object to write data to.
:raises: exception.ImageRefValidationFailed if source image file
doesn't exist.
:raises: exception.ImageDownloadFailed if exceptions were raised while
writing to file or creating hard link.
"""
source_image_path = self.validate_href(image_href)
dest_image_path = image_file.name
local_device = os.stat(dest_image_path).st_dev
try:
# We should have read and write access to source file to create
# hard link to it.
if (local_device == os.stat(source_image_path).st_dev and
os.access(source_image_path, os.R_OK | os.W_OK)):
image_file.close()
os.remove(dest_image_path)
os.link(source_image_path, dest_image_path)
else:
filesize = os.path.getsize(source_image_path)
with open(source_image_path, 'rb') as input_img:
sendfile.sendfile(image_file.fileno(), input_img.fileno(),
0, filesize)
except Exception as e:
raise exception.ImageDownloadFailed(image_href=image_href,
reason=e)
def copy_data(data_length, blocksize, infp, outfp):
# type: (int, int, BinaryIO, BinaryIO) -> None
'''
A utility function to copy data from the input file object to the output
file object. This function will use the most efficient copy method available,
which is often sendfile.
Parameters:
data_length - The amount of data to copy.
blocksize - How much data to copy per iteration.
infp - The file object to copy data from.
outfp - The file object to copy data to.
Returns:
Nothing.
'''
use_sendfile = False
if have_sendfile:
# Python 3 implements the fileno method for all file-like objects, so
# we can't just use the existence of the method to tell whether it is
# available. Instead, we try to assign it, and if we fail, then we
# assume it is not available.
try:
x_unused = infp.fileno() # NOQA
y_unused = outfp.fileno() # NOQA
use_sendfile = True
except (AttributeError, io.UnsupportedOperation):
pass
if use_sendfile:
# This is one of those instances where using the file object and the
# file descriptor causes problems. The sendfile() call actually updates
# the underlying file descriptor, but the file object does not know
# about it. To get around this, we instead get the offset, allow
# sendfile() to update the offset, then manually seek the file object
# to the right location. This ensures that the file object gets updated
# properly.
in_offset = infp.tell()
out_offset = outfp.tell()
sendfile(outfp.fileno(), infp.fileno(), in_offset, data_length)
infp.seek(in_offset + data_length)
outfp.seek(out_offset + data_length)
else:
left = data_length
readsize = blocksize
while left > 0:
if left < readsize:
readsize = left
data = infp.read(readsize)
# We have seen ISOs in the wild (Tribes Vengeance 1of4.iso) that
# lie about the size of their files, causing reads to fail (since
# we hit EOF before the supposed end of the file). If we are using
# sendfile above, sendfile just silently returns as much data as it
# can, with no additional checking. We should do the same here, so
# if we got less data than we asked for, abort the loop silently.
data_len = len(data)
if data_len != readsize:
data_len = left
outfp.write(data)
left -= data_len
def handle (self): if sendfile.has_sf_hdtr: #print sendfile.sendfile(self.request.fileno(), datafile.fileno(), 0, 0, (["HTTP/1.1 200 OK\r\n", "Content-Type: text/html\r\n", "Connection: close\r\n\r\n"], 'test')) print sendfile.sendfile(self.request.fileno(), datafile.fileno(), 0, 0, ["HTTP/1.1 200 OK\r\n", "Content-Type: text/html\r\n", "Connection: close\r\n\r\n"]) #print sendfile.sendfile(self.request.fileno(), datafile.fileno(), 0, 0, "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nConnection: close\r\n\r\n", 'test') else: print sendfile.sendfile(self.request.fileno(), datafile.fileno(), 0, 0) self.request.close()
def test_invalid_offset(self):
try:
sendfile.sendfile(self.sockno, self.fileno, -1, BUFFER_LEN)
except OSError:
err = sys.exc_info()[1]
self.assertEqual(err.errno, errno.EINVAL)
else:
self.fail("exception not raised")
def test_trailer(self):
with open(TESTFN2, "wb") as f:
f.write(b("abcde"))
with open(TESTFN2, "rb") as f:
sendfile.sendfile(self.sockno, f.fileno(), 0, trailer=b("12345"))
time.sleep(0.1)
self.client.close()
self.server.wait()
data = self.server.handler_instance.get_data()
self.assertEqual(data, b("abcde12345"))
def custom_resource(request, user_id, res_type):
if request.user.id != int(user_id):
return HttpResponseForbidden("You are not allowed to view this resource.")
customization = get_object_or_404(UserCustomization, user_id=user_id)
if res_type == "css":
if customization.custom_css:
return sendfile.sendfile(request, customization.custom_css.path)
elif res_type == "js":
if customization.custom_js:
return sendfile.sendfile(request, customization.custom_js.path)
return HttpResponseNotFound("Requested resource does not exist.")
def write():
if not lcls.strm:
yield gen.Task(reconnect)
if lcls.strm.closed():
yield gen.Task(reconnect)
strm = lcls.strm
if strm.closed():
print('failed to connect')
return
yield gen.Task(write_string, strm, b"Hello, world")
def on_queue_change(change):
lcls.q_len += change
print("Write queue size:", lcls.q_len)
strm.on_queue_change = on_queue_change
#fname = '/home/muravyev/opt/bl/f451/git/show_tv/test-data/dump1.txt'
fname = os.path.expanduser("~/opt/bl/f451/tmp/test_src/pervyj-720x406.ts")
sz = os.stat(fname).st_size
a_str = b"abcbx"
cnt = 3
yield gen.Task(write_number, strm, cnt*(sz + len(a_str)))
for i in range(cnt):
sendfile(strm, fname, sz)
strm.write(a_str)
with open(fname, "rb") as f:
m = make_hash_obj()
txt = f.read()
for i in range(cnt):
m.update(txt)
m.update(a_str)
yield gen.Task(finish_packet, strm, m)
# запись подряд 2х строк
s1 = b"|".join([b"aaaa" for i in range(1000000)])
s11 = s1 + s1
m = make_hash_obj()
m.update(s11)
yield gen.Task(write_number, strm, len(s11))
strm.write(s1)
strm.write(s1)
yield gen.Task(finish_packet, strm, m)
yield gen.Task(write_string, strm, b"Bye, world")
# конец передачи данных
yield gen.Task(write_number, strm, 0)
close_and_one_second_pause(strm)
def write_chunk(stream, chunk_fpath, payloadlen, prefix):
stream.write(prefix)
use_sendfile = configuration.use_sendfile
if use_sendfile:
sendfile(stream, chunk_fpath, payloadlen)
else:
with open(chunk_fpath, 'rb') as f:
stream.write(f.read())
if stream.closed():
logger.error("Write to DVR failed")
def test_non_socket(self):
fd_in = open(TESTFN, 'rb')
fd_out = open(TESTFN2, 'wb')
try:
sendfile.sendfile(fd_in.fileno(), fd_out.fileno(), 0, BUFFER_LEN)
except OSError:
err = sys.exc_info()[1]
self.assertEqual(err.errno, errno.EBADF)
else:
self.fail("exception not raised")
finally:
fd_in.close()
fd_out.close()
def test_non_socket(self):
fd_in = open(TESTFN, 'rb')
fd_out = open(TESTFN2, 'wb')
try:
sendfile.sendfile(fd_in.fileno(), fd_out.fileno(), 0, BUFFER_LEN)
except OSError:
err = sys.exc_info()[1]
self.assertEqual(err.errno, errno.EBADF)
else:
self.fail("exception not raised")
finally:
fd_in.close()
fd_out.close()
def test_trailer(self):
with open(TESTFN2, 'wb') as f:
f.write(b("abcde"))
with open(TESTFN2, 'rb') as f:
sendfile.sendfile(self.sockno,
f.fileno(),
0,
trailer=b("12345"))
time.sleep(.1)
self.client.close()
self.server.wait()
data = self.server.handler_instance.get_data()
self.assertEqual(data, b("abcde12345"))
def copy_data(data_length, blocksize, infp, outfp):
'''
A utility function to copy data from the input file object to the output
file object. This function will use the most efficient copy method available,
which is often sendfile.
Parameters:
data_length - The amount of data to copy.
blocksize - How much data to copy per iteration.
infp - The file object to copy data from.
outfp - The file object to copy data to.
Returns:
Nothing.
'''
use_sendfile = False
if have_sendfile:
# Python 3 implements the fileno method for all file-like objects, so
# we can't just use the existence of the method to tell whether it is
# available. Instead, we try to assign it, and if we fail, then we
# assume it is not available.
try:
x_unused = infp.fileno() # NOQA
y_unused = outfp.fileno() # NOQA
use_sendfile = True
except (AttributeError, io.UnsupportedOperation):
pass
if use_sendfile:
# This is one of those instances where using the file object and the
# file descriptor causes problems. The sendfile() call actually updates
# the underlying file descriptor, but the file object does not know
# about it. To get around this, we instead get the offset, allow
# sendfile() to update the offset, then manually seek the file object
# to the right location. This ensures that the file object gets updated
# properly.
in_offset = infp.tell()
out_offset = outfp.tell()
sendfile(outfp.fileno(), infp.fileno(), in_offset, data_length)
infp.seek(in_offset + data_length)
outfp.seek(out_offset + data_length)
else:
left = data_length
readsize = blocksize
while left > 0:
if left < readsize:
readsize = left
data = infp.read(readsize)
if len(data) != readsize:
raise pycdlibexception.PyCdlibInternalError("Failed to read expected bytes")
outfp.write(data)
left -= readsize
def copyfile(self, source, outputfile, closeSource=True):
if sendfile and isinstance(source, file):
# use sendfile(2) for high performance static file serving
sfileno = source.fileno()
size = os.fstat(sfileno).st_size
outputfile.flush()
sendfile(outputfile.fileno(), sfileno, 0, size)
source.close()
else:
# source is not a true file or sendfile(2) is unavailable, use regular write method
try:
shutil.copyfileobj(source, outputfile, length=32 * 1024)
except socket.error, x:
log.error("Socket error during copyfile: " + str(x))
def copyfile(self, source, outputfile, closeSource=True):
if sendfile and isinstance(source, file):
# use sendfile(2) for high performance static file serving
sfileno=source.fileno()
size=os.fstat(sfileno).st_size
outputfile.flush()
sendfile(outputfile.fileno(), sfileno, 0, size)
source.close()
else:
# source is not a true file or sendfile(2) is unavailable, use regular write method
try:
shutil.copyfileobj(source, outputfile, length=32*1024)
except socket.error,x:
log.error("Socket error during copyfile: "+str(x))
def archive_download(request, archive_uid=None):
try:
archive\
= models.ExperimentDataExport.objects.get(short_uid=archive_uid)
return sendfile(request,
filename=archive.filepath,
mimetype='application/x-tar',
attachment=True,
attachment_filename=archive.attachment_filename)
except ObjectDoesNotExist as e:
raise Http404(
"data archive {archive_name} does not exist.".format(archive_uid)
)
except MultipleObjectsReturned as e:
logger.warning('Multiple objects returned: %s' % e.message)
raise Http404(
"Can not return data archive {archive_name}.".format(archive_uid)
)
def get(self, request, *args, **kwargs):
album = self.get_object()
reuse_zip = AlbumDownload.objects.filter(
album=album, timestamp__gte=album.last_upload, failed=False
).exists()
filename = os.path.join(settings.SENDFILE_ROOT, 'albums',
str(album.user_id), str(album.id),
'{}.zip'.format(album.id))
if not reuse_zip:
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
download = AlbumDownload.objects.create(album=album, downloader=request.user, failed=True)
with zipfile.ZipFile(filename, 'w') as ziph:
for photo in album.photo_set.filter(trash=False):
if not photo.exists:
logger.warn('Missing photo: %d' % photo.id)
continue
image = photo.image.path
ziph.write(image, os.path.split(image)[1])
download.failed = False
download.save()
return sendfile(request, filename, attachment=True)
def get_file(request, slug, attachment_id, filename): attachment = get_object_or_404(Attachment, id=attachment_id, page__slug=slug) as_attachment = ((not imghdr.what(attachment.file.path) and 'embed' not in request.GET) or 'as_attachment' in request.GET) # ref https://github.com/johnsensible/django-sendfile return sendfile(request, attachment.file.path, attachment=as_attachment, attachment_filename=text_type(attachment))
def student_files(request, fileid, distid=''):
"""
Student file, uploaded by student as professionalskill. Model in students-app
Type3 and 4 (support and profskill) staff can see all studentfiles.
Responsible and assistant of student can view files.
Student itself can view its own files
:param request:
:param fileid: id of the student file.
:param distid: id of the distribution of the student.
:return:
"""
# first try PK, then filename
try:
obj = StudentFile.objects.get(id=fileid)
except:
# accessed by distribution, then by filename, (the old way)
obj = get_object_or_404(StudentFile, File='dist_{}/{}'.format(distid, fileid))
if get_grouptype("3") in request.user.groups.all() \
or Group.objects.get(name='type4staff') in request.user.groups.all() \
or obj.Distribution.Proposal.ResponsibleStaff == request.user \
or request.user in obj.Distribution.Proposal.Assistants.all() \
or obj.Distribution.Student == request.user:
# Allowed to view this file
return sendfile(request, obj.File.path, attachment=True, attachment_filename=obj.OriginalName)
else:
# not allowed
raise PermissionDenied("You are not allowed to view this file.")
def serve_media(request, path):
if path.startswith("videos/"):
if not request.user:
return HttpResponseForbidden()
user = request.user
if not user.is_active or not user.is_authenticated() or not user.is_staff:
return HttpResponseForbidden()
if path.startswith("videos/screens/"):
try:
sources = Source.objects.filter(thumbnails__name=path).values_list("name")[0]
except IndexError:
sources = ()
try:
screen = VideoScreen.objects.get(Q(image__in=sources) | Q(image=path))
if not user.is_superuser and screen.owner != user:
return HttpResponseForbidden()
except VideoScreen.DoesNotExist:
return HttpResponseNotFound()
elif path.startswith("videos/converted/"):
try:
video = ConvertedVideo.objects.get(video=path)
if not user.is_superuser and video.owner != user:
return HttpResponseForbidden()
except ConvertedVideo.DoesNotExist:
return HttpResponseNotFound()
else:
try:
video = WebVideo.objects.get(video=path)
if not user.is_superuser and video.owner != user:
return HttpResponseForbidden()
except WebVideo.DoesNotExist:
return HttpResponseNotFound()
return sendfile(request, os.path.join(settings.MEDIA_ROOT, path))
def export_download(request, uuid, token):
log = logging.getLogger('exporter.views.export_download')
log.info('Download Request by: %s' % (request.user.username))
export = get_object_or_404(Export, uuid=uuid)
#version = 'base'
print 'EXPORT: %s' % export
download_permission = False
if request.user == export.user and token == export.token:
download_permission = True
if not download_permission:
return HttpResponseForbidden('forbidden')
filename = '%s.%s' % (export.filename, 'zip')
export.set_downloaded()
return sendfile(request, export.file.path, attachment=True, attachment_filename=filename)
def serve_local(request: HttpRequest, path_id: str) -> HttpResponse:
local_path = get_local_file_path(path_id)
if local_path is None:
return HttpResponseNotFound('<p>File not found</p>')
# Here we determine whether a browser should treat the file like
# an attachment (and thus clicking a link to it should download)
# or like a link (and thus clicking a link to it should display it
# in a browser tab). This is controlled by the
# Content-Disposition header; `django-sendfile2` sends the
# attachment-style version of that header if and only if the
# attachment argument is passed to it. For attachments,
# django-sendfile2 sets the response['Content-disposition'] like
# this: `attachment; filename="zulip.txt"; filename*=UTF-8''zulip.txt`.
# The filename* parameter is omitted for ASCII filenames like this one.
#
# The "filename" field (used to name the file when downloaded) is
# unreliable because it doesn't have a well-defined encoding; the
# newer filename* field takes precedence, since it uses a
# consistent format (urlquoted). For more details on filename*
# and filename, see the below docs:
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition
mimetype, encoding = guess_type(local_path)
attachment = mimetype not in INLINE_MIME_TYPES
return sendfile(request, local_path, attachment=attachment,
mimetype=mimetype, encoding=encoding)
def download(self,
request,
pk=None,
download_key=None,
download_job_id=None):
try:
self.get_queryset().get(pk=pk)
except QueryJob.DoesNotExist:
raise NotFound
download_config = get_download_config(download_key)
if download_config is None:
raise ValidationError({
'download':
'Download key "{}" is not supported.'.format(download_key)
})
download_job_model = import_class(download_config['model'])
try:
download_job = download_job_model.objects.get(query_job=pk,
pk=download_job_id)
except download_job_model.DoesNotExist:
raise NotFound
if download_job.phase == download_job.PHASE_COMPLETED and request.GET.get(
'download', True):
return sendfile(request, download_job.file_path, attachment=True)
else:
return Response(download_job.phase)
def serve_local(request: HttpRequest, path_id: str) -> HttpResponse:
local_path = get_local_file_path(path_id)
if local_path is None:
return HttpResponseNotFound('<p>File not found</p>')
# Here we determine whether a browser should treat the file like
# an attachment (and thus clicking a link to it should download)
# or like a link (and thus clicking a link to it should display it
# in a browser tab). This is controlled by the
# Content-Disposition header; `django-sendfile` sends the
# attachment-style version of that header if and only if the
# attachment argument is passed to it. For attachments,
# django-sendfile sets the response['Content-disposition'] like
# this: `attachment; filename="b'zulip.txt'"; filename*=UTF-8''zulip.txt`.
#
# The "filename" field (used to name the file when downloaded) is
# unreliable because it doesn't have a well-defined encoding; the
# newer filename* field takes precedence, since it uses a
# consistent format (urlquoted). For more details on filename*
# and filename, see the below docs:
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition
attachment = True
file_type = guess_type(local_path)[0]
if file_type is not None and (file_type.startswith("image/") or
file_type == "application/pdf"):
attachment = False
return sendfile(request, local_path, attachment=attachment)
def export(request, activityId):
activity = AuditActivity.objects.get(pk=activityId)
path, filename = _export(activity)
return sendfile(request,
path,
attachment=True,
attachment_filename=filename)
def download_file(request, order_number=None, hashcode=None, product_pk=None):
""
product = get_object_or_404(Product, pk=product_pk)
order = get_object_or_404(Order, number=order_number)
if hashcode != order.verification_hash():
raise Http404()
try:
referer = request.META['HTTP_REFERER']
path = urlparse.urlparse(referer).path
match = resolve(path)
match_hashcode = match.kwargs.get('hashcode')
match_order_number = match.kwargs.get('order_number')
order = get_object_or_404(Order, number=match_order_number)
if match_hashcode != order.verification_hash():
raise Http404()
if match_hashcode != hashcode or match_order_number != order_number:
raise Http404()
return sendfile(request, product.filename.path, attachment=True)
except KeyError:
raise Http404()
def get(self, request, *args, **kwargs):
self.object = self.get_object()
if isinstance(self.object, SectionImage):
f = self.object.image
elif isinstance(self.object, SectionFile):
f = self.object.file
return sendfile(request, f.path)
def get(self, request, format=None):
base_url = request.build_absolute_uri().replace(
reverse('api:export-playlist'), "")
playlists = Playlist.objects.filter(owner=request.user)
playlists_zip_directory = '/tmp' + settings.SENDFILE_ROOT
playlists_zip_path = os.path.join(playlists_zip_directory,
request.user.username + '.zip')
if not os.path.exists(playlists_zip_directory):
os.makedirs(playlists_zip_directory)
with zipfile.ZipFile(playlists_zip_path, 'w') as playlists_zip:
for playlist in playlists:
tracks = playlist.track_set.all()
tracks_url = [
base_url + reverse('api:stream', args=['']) +
str(track.file.relative_path()) for track in tracks
]
playlist_string = '\n'.join(tracks_url)
playlists_zip.writestr(playlist.name + '.m3u', playlist_string)
return sendfile(
request,
playlists_zip_path,
root_url='/tmp' + settings.SENDFILE_URL,
root_directory='/tmp' + settings.SENDFILE_ROOT,
attachment=True,
attachment_filename='playlists.zip',
mimetype='application/x-zip-compressed',
)
def download_pdf(request, contest=None, slug=None, attach=True, **kwargs):
problem = get_object_or_404(models.Problem, contest__slug=contest, slug=slug)
if problem.contest.has_ended() or request.user.is_authenticated() and problem.contest.has_contestant(request.user):
return sendfile(request, problem.pdf.path, attachment=attach, attachment_filename=slug + ".pdf")
return HttpResponseNotFound("404 Not Found")
def dump(self, outfile):
"""Dumps the content of the image into a file.
This method will only dump the actual payload, found by reading the
partition table. Empty space in the end of the device will be ignored.
"""
MB = 2**20
blocksize = 2**22 # 4MB
progr_size = (self.size + MB - 1) // MB # in MB
progressbar = self.out.Progress(progr_size, "Dumping image file", 'mb')
with self.raw_device() as raw:
with open(raw, 'rb') as src:
with open(outfile, "wb") as dst:
left = self.size
offset = 0
progressbar.next()
while left > 0:
length = min(left, blocksize)
sent = sendfile(dst.fileno(), src.fileno(), offset,
length)
# Workaround for python-sendfile API change. In
# python-sendfile 1.2.x (py-sendfile) the returning
# value of sendfile is a tuple, where in version 2.x
# (pysendfile) it is just a single integer.
if isinstance(sent, tuple):
sent = sent[1]
offset += sent
left -= sent
progressbar.goto((self.size - left) // MB)
progressbar.success('image file %s was successfully created' % outfile)
def _cat_amd64(filepath, out_fd, use_sendfile):
if use_sendfile:
return open_file(filepath), sendfile('rax', out_fd)
else:
return (open_file(filepath), "xchg ebp, eax\ncat_helper1:",
read_stack('rbp', 48, False), "test eax, eax\njle cat_helper2",
write_stack(out_fd, 'rax'), "jmp cat_helper1\ncat_helper2:")
def _export_annotations(db_task, rq_id, request, format_name, action, callback, filename):
if action not in {"", "download"}:
raise serializers.ValidationError(
"Unexpected action specified for the request")
format_desc = {f.DISPLAY_NAME: f
for f in dm.views.get_export_formats()}.get(format_name)
if format_desc is None:
raise serializers.ValidationError(
"Unknown format specified for the request")
elif not format_desc.ENABLED:
return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)
queue = django_rq.get_queue("default")
rq_job = queue.fetch_job(rq_id)
if rq_job:
last_task_update_time = timezone.localtime(db_task.updated_date)
request_time = rq_job.meta.get('request_time', None)
if request_time is None or request_time < last_task_update_time:
rq_job.cancel()
rq_job.delete()
else:
if rq_job.is_finished:
file_path = rq_job.return_value
if action == "download" and osp.exists(file_path):
rq_job.delete()
timestamp = datetime.strftime(last_task_update_time,
"%Y_%m_%d_%H_%M_%S")
filename = filename or \
"task_{}-{}-{}{}".format(
db_task.name, timestamp,
format_name, osp.splitext(file_path)[1])
return sendfile(request, file_path, attachment=True,
attachment_filename=filename.lower())
else:
if osp.exists(file_path):
return Response(status=status.HTTP_201_CREATED)
elif rq_job.is_failed:
exc_info = str(rq_job.exc_info)
rq_job.delete()
return Response(exc_info,
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
else:
return Response(status=status.HTTP_202_ACCEPTED)
try:
if request.scheme:
server_address = request.scheme + '://'
server_address += request.get_host()
except Exception:
server_address = None
ttl = dm.views.CACHE_TTL.total_seconds()
queue.enqueue_call(func=callback,
args=(db_task.id, format_name, server_address), job_id=rq_id,
meta={ 'request_time': timezone.localtime() },
result_ttl=ttl, failure_ttl=ttl)
return Response(status=status.HTTP_202_ACCEPTED)
def download_bylaw(request, bylaw): """ View for downloading bylaws """ bylawObject = Bylaws.objects.get(pk=bylaw) return sendfile(request, bylawObject.filepath.path, attachment=True, attachment_filename="Bylaws " + str(bylawObject.date))
def media(request, path):
file_path = os.path.join(MEDIA_ROOT, path)
if len(request.GET) > 0:
if not what(file_path):
return sendfile(request, file_path)
if 'w' in request.GET:
_file_path = image_resize(file_path, 'w', request.GET.get('w', 600))
if 'h' in request.GET:
_file_path = image_resize(file_path, 'h', request.GET.get('h', 600))
if 's' in request.GET:
_file_path = image_resize(file_path, 's', request.GET.get('s', 600))
if _file_path:
file_path = _file_path
if not os.path.exists(file_path):
file_path = os.path.join(STATIC_DIR, 'img/image-not-found.jpg')
return sendfile(request, file_path)
def get(self, request, *args, **kwargs):
# Get document or raise HTTP404
doc = get_object_or_404(models.Document, pk=self.kwargs.get('pk'))
# If document is an exam or marked as non-public, require login
if doc.dtype == doc.DTypes.EXAM or doc.public is False:
if not self.request.user.is_authenticated():
return redirect(
'%s?next=%s' %
(reverse('auth_login'),
reverse('documents:document_list',
kwargs={'category': slugify(doc.category.name)})))
# Log download
ip = helpers.get_client_ip(request)
timerange = datetime.datetime.now() - datetime.timedelta(1)
filters = {'document': doc, 'ip': ip, 'timestamp__gt': timerange}
if not models.DocumentDownload.objects.filter(**filters).exists():
models.DocumentDownload.objects.create(document=doc, ip=ip)
# Serve file
filename = unicodedata.normalize('NFKD', doc.original_filename) \
.encode('us-ascii', 'ignore')
attachment = not filename.lower().endswith('.pdf')
return sendfile(request,
doc.document.path,
attachment=attachment,
attachment_filename=filename)
def index(request):
r = glob.glob('./tmp/spectrum_v17/*')
for file in r:
os.remove(file)
if request.method == 'POST':
form = SpectrumForm(request.POST, request.FILES)
if form.is_valid():
new_doc = Document(docfile=request.FILES['docfile'])
new_doc.save()
input_file = glob.glob('./tmp/spectrum_v17/*')[0]
date_str = str(datetime.now())[:-7].replace(':',
'-').replace(' ', '-')
output_file = './tmp/spectrum_v17/' + date_str + '.zip'
spectrum_v17.prepare_spectrum_new.prepare(input_file, output_file)
return sendfile(request, output_file, attachment=True)
else:
form = SpectrumForm()
return render(request, 'spectrum_v17/spectrum_index.html', {'form': form})
def get(self, request, *args, **kwargs):
album = self.get_object()
reuse_zip = AlbumDownload.objects.filter(
album=album, timestamp__gte=album.last_upload,
failed=False).exists()
filename = os.path.join(settings.SENDFILE_ROOT, 'albums',
str(album.user_id), str(album.id),
'{}.zip'.format(album.id))
if not reuse_zip:
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
download = AlbumDownload.objects.create(album=album,
downloader=request.user,
failed=True)
with zipfile.ZipFile(filename, 'w') as ziph:
for photo in album.photo_set.filter(trash=False):
if not photo.exists:
logger.warn('Missing photo: %d' % photo.id)
continue
image = photo.image.path
ziph.write(image, os.path.split(image)[1])
download.failed = False
download.save()
return sendfile(request, filename, attachment=True)
def download_file(request, share, subpath=None):
from sendfile import sendfile
share.last_data_access = timezone.now()
share.save()
file_path = os.path.join(share.get_path(), subpath)
response = {'path': file_path}
return sendfile(request, os.path.realpath(file_path))
def dump(self, outfile):
"""Dumps the content of the image into a file.
This method will only dump the actual payload, found by reading the
partition table. Empty space in the end of the device will be ignored.
"""
MB = 2 ** 20
blocksize = 2 ** 22 # 4MB
progr_size = (self.size + MB - 1) // MB # in MB
progressbar = self.out.Progress(progr_size, "Dumping image file", 'mb')
with self.raw_device() as raw:
with open(raw, 'rb') as src:
with open(outfile, "wb") as dst:
left = self.size
offset = 0
progressbar.next()
while left > 0:
length = min(left, blocksize)
sent = sendfile(dst.fileno(), src.fileno(), offset,
length)
# Workaround for python-sendfile API change. In
# python-sendfile 1.2.x (py-sendfile) the returning
# value of sendfile is a tuple, where in version 2.x
# (pysendfile) it is just a single integer.
if isinstance(sent, tuple):
sent = sent[1]
offset += sent
left -= sent
progressbar.goto((self.size - left) // MB)
progressbar.success('image file %s was successfully created' % outfile)
def initiate_send (self):
if self._sendfile is None:
async_chat.initiate_send (self)
else:
if len(self.ac_out_buffer):
async_chat.initiate_send (self)
else:
fd, offset, bytes, callback = self._sendfile
me = self.socket.fileno()
try:
sent = sendfile.sendfile (fd, me, offset, bytes)
offset = offset + sent
bytes = bytes - sent
if bytes:
self._sendfile = (fd, offset, bytes, callback)
else:
self._sendfile = None
self.ac_out_buffer_size = self._saved_obs
if callback is not None:
# success
callback (1, fd)
except:
self._sendfile = None
self.ac_out_buffer_size = self._saved_obs
# failure
if callback is not None:
callback (0, fd)
def exportCustomers(request):
customers = Customer.objects.all()
f = '/tmp/{}.xls'.format(str(uuid.uuid4()))
xf = xlwt.Workbook()
sheet = xf.add_sheet('sheet1')
titles = [
'客户名称', '评级', '主要股东信息', '法人', '注册资本(万元)', '成立年份', '成立年限', '公司类型',
'公司性质', '地址信息', '备注'
]
for index, title in enumerate(titles):
sheet.write(0, index, title)
props = [
'name', 'rating', 'shareholder', 'faren', 'capital', 'year',
'nianxian', 'displayCategory', 'displayNature', 'address', 'desc'
]
for row, customer in enumerate(customers):
for col, prop in enumerate(props):
sheet.write(row + 1, col, getattr(customer, prop))
xf.save(f)
return sendfile(request,
f,
attachment=True,
attachment_filename='export.xls')
def render_to_response(self, context, **response_kwargs):
dump_format = self.request.GET.get('format')
job_id = self.request.GET.get('job')
if not job_id:
return HttpResponseRedirect(reverse_lazy('system:diagnostic'))
result = dump_diagnostics.AsyncResult(job_id)
if result.state == 'SUCCESS':
if not self.request.is_ajax() and dump_format == 'csv':
file_path = os.path.join(default_storage.location,
result.info['file_path'])
# Encoding must be specified to prevent guessing, which then results in 'gzip' encoding
# being sent to the browser, and causes the browser to decompress the content on-the-fly,
# therefore corrupting the attachment.
return sendfile(self.request,
file_path,
attachment=True,
mimetype='application/octet-stream',
encoding='identity')
download_link = '%s?job=%s&format=csv' % (
reverse_lazy('system:diagnostic-download'), job_id)
context['progress'] = 100
context['progress_active'] = False
context['status'] = (
'Your download is ready. Click <a href="%s">here</a> ' +
'if the download does not start automatically.'
) % download_link
context['download_link'] = download_link
elif result.state in ['FAILURE', 'REVOKED']:
context['progress'] = 99
context['progress_active'] = False
if result.state == 'REVOKED':
context['status'] = ((
'Too many download requests at a time. ' +
'Please wait one (1) minute and <a href="%s">try</a> again.'
) % reverse_lazy('system:diagnostic'))
else:
context['status'] = ((
'Some error has occurred. Please <a href="%s">try</a> again.<br/>'
+ '<small>%s</small>') %
(reverse_lazy('system:diagnostic'),
result.info))
else:
if result.state == 'PROGRESS':
context['progress'] = result.info['progress']
else:
context['progress'] = 0
context['progress_active'] = True
context[
'status'] = 'Please wait while the system prepares your download.'
if context['progress_active']:
response_kwargs['status'] = 202
return super(DiagnosticDownloadView,
self).render_to_response(context, **response_kwargs)
def write(filepath, target):
"""
Copy a large file while reporting progress.
This function uses sendfile() to achieve fast copying of data.
"""
logging.log(logging.INFO, "Writing {} to {}.".format(filepath, target))
device = False
if target.startswith("/dev"):
device = True
if device:
logging.log(logging.INFO,
"Target is a device, will attempt to unmount.")
for disk in usbdisks.get_disks():
if disk.fs_path.startswith(target):
logging.log(logging.INFO,
"Unmounting {}.".format(disk.fs_path))
os.system("umount " + disk.fs_path)
filesize = os.path.getsize(filepath)
fin = open(filepath, 'rb')
fout = open(target, 'wb')
offset = 0
while True:
copied = sendfile(fout.fileno(), fin.fileno(), offset, BUFFSIZE)
if not copied: break
offset += copied
yield copied
if device:
os.system('udisksctl power-off -b ' + target)
def stream_html5(request, uuid):
media = get_object_or_404(Media, uuid=uuid)
stream_permission = False
if request.user and request.user.has_perm('alibrary.play_media'):
stream_permission = True
# check if unrestricted license
#if not stream_permission:
# if media.license and media.license.restricted == False:
# stream_permission = True
if not stream_permission:
log.warning('unauthorized attempt by "%s" to download: %s - "%s"' %
(request.user.username if request.user else 'unknown',
media.pk, media.name))
raise PermissionDenied
try:
from atracker.util import create_event
create_event(request.user, media, None, 'stream')
except:
pass
media_file = media.get_cache_file('mp3', 'base')
if not media_file:
return HttpResponseBadRequest('unable to get cache file')
return sendfile(request, media_file)
def send_whole_file(sock_fd, file_fd):
offset = 0
while True:
sent = sendfile(sock_fd, file_fd, offset, 65536)
if sent == 0:
break
offset += sent
def send_file(self, file_name, buffer_size=4096):
"""
:param file_name: file path
:return: response_body, total_response_size
"""
if 'linux' not in sys.platform.lower():
raise Exception('sendfile system call only available on linux.')
try:
self.conn.send(self.buf)
resp_header = self.conn.recv(self.header.resp_header_len())
self.header.unpack_resp(resp_header)
if self.header.status != 0:
raise Exception('Error: %d, %s' % (self.header.status, os.strerror(self.header.status)))
sock_fd = self.conn.get_fd()
offset = 0
with open(file_name, "rb") as f_obj:
while 1:
try:
sent = sendfile(sock_fd, f_obj.fileno(), offset, buffer_size)
if sent == 0:
break
offset += sent
except OSError as e:
if e.errno == errno.EAGAIN:
continue
raise e
resp_body = self.conn.recv(self.header.resp_pkg_len)
return resp_body, self.header.resp_pkg_len
except Exception as e:
if self.conn:
self.conn.disconnect()
raise e
finally:
del self.conn
def __iter__(self):
class OfLength:
def __init__(self, len):
self.len = len
def __len__(self):
return self.len
while self.sending:
try:
sent = sendfile.sendfile(self.connection.sock.fileno(),
self.body.fileno(),
self.offset,
CHUNKSIZE)
except OSError as e:
# suprisingly, sendfile may fail transiently instead of
# blocking, in which case we select on the socket in order
# to wait on its return to a writeable state before resuming
# the send loop
if e.errno in (errno.EAGAIN, errno.EBUSY):
wlist = [self.connection.sock.fileno()]
rfds, wfds, efds = select.select([], wlist, [])
if wfds:
continue
raise
self.sending = (sent != 0)
self.offset += sent
yield OfLength(sent)
def run(self):
while 1:
event = self.queue.get()
if check_filetype(event.pathname):
print time.asctime(),event.maskname, event.pathname
filepath = event.path.split('/')[-1:][0]
filename = event.name
filesize = os.stat(os.path.join(event.path, filename)).st_size
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
filepath_len = len(filepath)
filename_len = len(filename)
sock.connect(self.addr)
offset = 0
data = struct.pack("!LL128s128sL",filepath_len, filename_len, filepath,filename,filesize)
fd = open(event.pathname,'rb')
sock.sendall(data)
if "sendfile" in sys.modules:
# print "use sendfile(2)"
while 1:
sent = sendfile(sock.fileno(), fd.fileno(), offset, self.chunk_size)
if sent == 0:
break
offset += sent
else:
# print "use original send function"
while 1:
data = fd.read(self.chunk_size)
if not data: break
sock.send(data)
sock.close()
fd.close()
def page_ocr_xml(request, lccn, date, edition, sequence):
title, issue, page = _get_tip(lccn, date, edition, sequence)
if page.ocr_abs_filename:
response = sendfile(request, page.ocr_abs_filename)
return add_cache_tag(response, "lccn=%s" % lccn)
else:
raise Http404("No ocr for page %s" % page)
def media_download(request, slug, format, version):
media = get_object_or_404(Media, slug=slug)
version = 'base'
download_permission = False
if not download_permission:
return HttpResponseForbidden('forbidden')
if format in ['mp3', 'flac', 'wav']:
cache_file = media.get_cache_file(format, version).path
else:
raise Http404
filename = '%02d %s - %s' % (media.tracknumber,
media.name.encode('ascii', 'ignore'),
media.artist.name.encode('ascii', 'ignore'))
filename = '%s.%s' % (filename, format)
return sendfile(request,
cache_file,
attachment=True,
attachment_filename=filename)
def stream(self, request, pk=None, format_key=None):
try:
job = self.get_queryset().get(pk=pk)
except QueryJob.DoesNotExist:
raise NotFound
try:
format_config = get_format_config(format_key)
except IndexError:
raise ValidationError({'format': "Not supported."})
try:
download_job = DownloadJob.objects.get(job=job,
format_key=format_key)
# check if the file was lost
if download_job.phase == download_job.PHASE_COMPLETED and os.path.isfile(
download_job.file_path):
# stream the previously created file
return sendfile(request, download_job.file_path)
except DownloadJob.DoesNotExist:
pass
# stream the table directly from the database
response = FileResponse(job.stream(format_key),
content_type=format_config['content_type'])
return response
def model(self, request, pk):
"""Download Trained Model Data."""
db_detector = self.get_object()
if db_detector.status != str(models.Status.TRAINED):
raise Http404('Model is not in TRAINED status. Current status: {}'.format(
db_detector.status))
queue = django_rq.get_queue("default")
rq_id = "{}@/api/opentpod/v1/detectors/{}/data".format(
self.request.user, pk)
rq_job = queue.fetch_job(rq_id)
if request.method == 'GET':
if rq_job:
if rq_job.is_finished:
return sendfile.sendfile(request, rq_job.meta["file_path"], attachment=True,
attachment_filename=str(
db_detector.get_export_file_path().name))
elif rq_job.is_failed:
exc_info = str(rq_job.exc_info)
rq_job.delete()
return Response(data=exc_info, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
else:
return Response(status=status.HTTP_202_ACCEPTED, data={json.dumps('created')})
else:
raise Http404
if request.method == 'POST':
rq_job = queue.enqueue_call(
func=bg_tasks.export,
args=(db_detector,),
job_id=rq_id,
)
rq_job.meta["file_path"] = db_detector.get_export_file_path()
rq_job.save_meta()
return Response(status=status.HTTP_202_ACCEPTED, data={json.dumps('created')})
def serve_local(request: HttpRequest, path_id: str) -> HttpResponse:
local_path = get_local_file_path(path_id)
if local_path is None:
return HttpResponseNotFound('<p>File not found</p>')
# Here we determine whether a browser should treat the file like
# an attachment (and thus clicking a link to it should download)
# or like a link (and thus clicking a link to it should display it
# in a browser tab). This is controlled by the
# Content-Disposition header; `django-sendfile` sends the
# attachment-style version of that header if and only if the
# attachment argument is passed to it. For attachments,
# django-sendfile sets the response['Content-disposition'] like
# this: `attachment; filename="b'zulip.txt'"; filename*=UTF-8''zulip.txt`.
#
# The "filename" field (used to name the file when downloaded) is
# unreliable because it doesn't have a well-defined encoding; the
# newer filename* field takes precedence, since it uses a
# consistent format (urlquoted). For more details on filename*
# and filename, see the below docs:
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition
attachment = True
file_type = guess_type(local_path)[0]
if file_type is not None and (file_type.startswith("image/")
or file_type == "application/pdf"):
attachment = False
return sendfile(request, local_path, attachment=attachment)
def __iter__(self):
class OfLength:
def __init__(self, len):
self.len = len
def __len__(self):
return self.len
while self.sending:
try:
sent = sendfile.sendfile(self.connection.sock.fileno(),
self.body.fileno(), self.offset,
CHUNKSIZE)
except OSError as e:
# suprisingly, sendfile may fail transiently instead of
# blocking, in which case we select on the socket in order
# to wait on its return to a writeable state before resuming
# the send loop
if e.errno in (errno.EAGAIN, errno.EBUSY):
wlist = [self.connection.sock.fileno()]
rfds, wfds, efds = select.select([], wlist, [])
if wfds:
continue
raise
self.sending = (sent != 0)
self.offset += sent
yield OfLength(sent)