def test_large_file(self): data = b'x' * (1024 * 600) req = Request.from_values(data={'foo': (BytesIO(data), 'test.txt')}, method='POST') # make sure we have a real file here, because we expect to be # on the disk. > 1024 * 500 self.assert_true(hasattr(req.files['foo'].stream, u'fileno')) # close file to prevent fds from leaking req.files['foo'].close()
def test_multi_part_line_breaks_bytes(): data = b'abcdef\r\nghijkl\r\nmnopqrstuvwxyz\r\nABCDEFGHIJK' test_stream = BytesIO(data) lines = list( wsgi.make_line_iter(test_stream, limit=len(data), buffer_size=16)) assert lines == [ b'abcdef\r\n', b'ghijkl\r\n', b'mnopqrstuvwxyz\r\n', b'ABCDEFGHIJK' ] data = b'abc\r\nThis line is broken by the buffer length.' \ b'\r\nFoo bar baz' test_stream = BytesIO(data) lines = list( wsgi.make_line_iter(test_stream, limit=len(data), buffer_size=24)) assert lines == [ b'abc\r\n', b'This line is broken by the buffer ' b'length.\r\n', b'Foo bar baz' ]
def test_multi_part_line_breaks_bytes(): data = b"abcdef\r\nghijkl\r\nmnopqrstuvwxyz\r\nABCDEFGHIJK" test_stream = BytesIO(data) lines = list(wsgi.make_line_iter(test_stream, limit=len(data), buffer_size=16)) assert lines == [ b"abcdef\r\n", b"ghijkl\r\n", b"mnopqrstuvwxyz\r\n", b"ABCDEFGHIJK", ] data = b"abc\r\nThis line is broken by the buffer length." b"\r\nFoo bar baz" test_stream = BytesIO(data) lines = list(wsgi.make_line_iter(test_stream, limit=len(data), buffer_size=24)) assert lines == [ b"abc\r\n", b"This line is broken by the buffer " b"length.\r\n", b"Foo bar baz", ]
def test_streamed_url_decoding(self): item1 = u'a' * 100000 item2 = u'b' * 400 string = ('a=%s&b=%s&c=%s' % (item1, item2, item2)).encode('ascii') gen = urls.url_decode_stream(BytesIO(string), limit=len(string), return_iterator=True) self.assert_strict_equal(next(gen), ('a', item1)) self.assert_strict_equal(next(gen), ('b', item2)) self.assert_strict_equal(next(gen), ('c', item2)) self.assert_raises(StopIteration, lambda: next(gen))
def default_stream_factory( total_content_length, filename, content_type, content_length=None ): """The stream factory that is used per default.""" max_size = 1024 * 500 if SpooledTemporaryFile is not None: return SpooledTemporaryFile(max_size=max_size, mode="wb+") if total_content_length is None or total_content_length > max_size: return TemporaryFile("wb+") return BytesIO()
def test_make_chunk_iter_bytes(): data = [b'abcdefXghi', b'jklXmnopqrstuvwxyzX', b'ABCDEFGHIJK'] rv = list(wsgi.make_chunk_iter(data, 'X')) assert rv == [b'abcdef', b'ghijkl', b'mnopqrstuvwxyz', b'ABCDEFGHIJK'] data = b'abcdefXghijklXmnopqrstuvwxyzXABCDEFGHIJK' test_stream = BytesIO(data) rv = list( wsgi.make_chunk_iter(test_stream, 'X', limit=len(data), buffer_size=4)) assert rv == [b'abcdef', b'ghijkl', b'mnopqrstuvwxyz', b'ABCDEFGHIJK']
def test_missing_multipart_boundary(self): data = (b'--foo\r\nContent-Disposition: form-field; name=foo\r\n\r\n' b'Hello World\r\n' b'--foo\r\nContent-Disposition: form-field; name=bar\r\n\r\n' b'bar=baz\r\n--foo--') req = Request.from_values(input_stream=BytesIO(data), content_length=len(data), content_type='multipart/form-data', method='POST') assert req.form == {}
def test_failures(self): def parse_multipart(stream, boundary, content_length): parser = formparser.MultiPartParser(content_length) return parser.parse(stream, boundary, content_length) self.assert_raises(ValueError, parse_multipart, BytesIO(), b'broken ', 0) data = b'--foo\r\n\r\nHello World\r\n--foo--' self.assert_raises(ValueError, parse_multipart, BytesIO(data), b'foo', len(data)) data = b'--foo\r\nContent-Disposition: form-field; name=foo\r\n' \ b'Content-Transfer-Encoding: base64\r\n\r\nHello World\r\n--foo--' self.assert_raises(ValueError, parse_multipart, BytesIO(data), b'foo', len(data)) data = b'--foo\r\nContent-Disposition: form-field; name=foo\r\n\r\nHello World\r\n' self.assert_raises(ValueError, parse_multipart, BytesIO(data), b'foo', len(data)) x = formparser.parse_multipart_headers(['foo: bar\r\n', ' x test\r\n']) self.assert_strict_equal(x['foo'], 'bar\n x test') self.assert_raises(ValueError, formparser.parse_multipart_headers, ['foo: bar\r\n', ' x test'])
def test_streamed_url_decoding(): item1 = u"a" * 100000 item2 = u"b" * 400 string = ("a=%s&b=%s&c=%s" % (item1, item2, item2)).encode("ascii") gen = urls.url_decode_stream( BytesIO(string), limit=len(string), return_iterator=True ) strict_eq(next(gen), ("a", item1)) strict_eq(next(gen), ("b", item2)) strict_eq(next(gen), ("c", item2)) pytest.raises(StopIteration, lambda: next(gen))
def loader(path): if path is None: return None, None path = posixpath.join(package_path, path) if not provider.has_resource(path): return None, None basename = posixpath.basename(path) if filesystem_bound: return basename, self._opener( provider.get_resource_filename(manager, path)) s = provider.get_resource_string(manager, path) return basename, lambda: (BytesIO(s), loadtime, len(s))
def test_bad_newline_bad_newline_assumption(self): class ISORequest(Request): charset = 'latin1' contents = b'U2vlbmUgbORu' data = b'--foo\r\nContent-Disposition: form-data; name="test"\r\n' \ b'Content-Transfer-Encoding: base64\r\n\r\n' + \ contents + b'\r\n--foo--' req = ISORequest.from_values(input_stream=BytesIO(data), content_length=len(data), content_type='multipart/form-data; boundary=foo', method='POST') self.assert_strict_equal(req.form['test'], u'Sk\xe5ne l\xe4n')
def test_empty_multipart(self): environ = {} data = b"--boundary--" environ["REQUEST_METHOD"] = "POST" environ["CONTENT_TYPE"] = "multipart/form-data; boundary=boundary" environ["CONTENT_LENGTH"] = str(len(data)) environ["wsgi.input"] = BytesIO(data) stream, form, files = parse_form_data(environ, silent=False) rv = stream.read() assert rv == b"" assert form == MultiDict() assert files == MultiDict()
def test_empty_multipart(self): environ = {} data = b'--boundary--' environ['REQUEST_METHOD'] = 'POST' environ['CONTENT_TYPE'] = 'multipart/form-data; boundary=boundary' environ['CONTENT_LENGTH'] = str(len(data)) environ['wsgi.input'] = BytesIO(data) stream, form, files = parse_form_data(environ, silent=False) rv = stream.read() assert rv == b'' assert form == MultiDict() assert files == MultiDict()
def test_file_no_content_type(self): data = ( b'--foo\r\n' b'Content-Disposition: form-data; name="test"; filename="test.txt"\r\n\r\n' b'file contents\r\n--foo--') data = Request.from_values( input_stream=BytesIO(data), content_length=len(data), content_type='multipart/form-data; boundary=foo', method='POST') assert data.files['test'].filename == 'test.txt' strict_eq(data.files['test'].read(), b'file contents')
def test_limiting(self): data = b'foo=Hello+World&bar=baz' req = Request.from_values(input_stream=BytesIO(data), content_length=len(data), content_type='application/x-www-form-urlencoded', method='POST') req.max_content_length = 400 self.assert_strict_equal(req.form['foo'], u'Hello World') req = Request.from_values(input_stream=BytesIO(data), content_length=len(data), content_type='application/x-www-form-urlencoded', method='POST') req.max_form_memory_size = 7 self.assert_raises(RequestEntityTooLarge, lambda: req.form['foo']) req = Request.from_values(input_stream=BytesIO(data), content_length=len(data), content_type='application/x-www-form-urlencoded', method='POST') req.max_form_memory_size = 400 self.assert_strict_equal(req.form['foo'], u'Hello World') data = (b'--foo\r\nContent-Disposition: form-field; name=foo\r\n\r\n' b'Hello World\r\n' b'--foo\r\nContent-Disposition: form-field; name=bar\r\n\r\n' b'bar=baz\r\n--foo--') req = Request.from_values(input_stream=BytesIO(data), content_length=len(data), content_type='multipart/form-data; boundary=foo', method='POST') req.max_content_length = 4 self.assert_raises(RequestEntityTooLarge, lambda: req.form['foo']) req = Request.from_values(input_stream=BytesIO(data), content_length=len(data), content_type='multipart/form-data; boundary=foo', method='POST') req.max_content_length = 400 self.assert_strict_equal(req.form['foo'], u'Hello World') req = Request.from_values(input_stream=BytesIO(data), content_length=len(data), content_type='multipart/form-data; boundary=foo', method='POST') req.max_form_memory_size = 7 self.assert_raises(RequestEntityTooLarge, lambda: req.form['foo']) req = Request.from_values(input_stream=BytesIO(data), content_length=len(data), content_type='multipart/form-data; boundary=foo', method='POST') req.max_form_memory_size = 400 self.assert_strict_equal(req.form['foo'], u'Hello World')
def test_file_no_content_type(self): data = ( b"--foo\r\n" b'Content-Disposition: form-data; name="test"; filename="test.txt"\r\n\r\n' b"file contents\r\n--foo--") data = Request.from_values( input_stream=BytesIO(data), content_length=len(data), content_type="multipart/form-data; boundary=foo", method="POST", ) assert data.files["test"].filename == "test.txt" strict_eq(data.files["test"].read(), b"file contents")
def test_nonstandard_line_endings(self): for nl in b'\n', b'\r', b'\r\n': data = nl.join( (b'--foo', b'Content-Disposition: form-data; name=foo', b'', b'this is just bar', b'--foo', b'Content-Disposition: form-data; name=bar', b'', b'blafasel', b'--foo--')) req = Request.from_values(input_stream=BytesIO(data), content_length=len(data), content_type='multipart/form-data; ' 'boundary=foo', method='POST') strict_eq(req.form['foo'], u'this is just bar') strict_eq(req.form['bar'], u'blafasel')
def test_extra_newline(self): # this test looks innocent but it was actually timeing out in # the Werkzeug 0.5 release version (#394) data = (b'\r\n\r\n--foo\r\n' b'Content-Disposition: form-data; name="foo"\r\n\r\n' b'a string\r\n' b'--foo--') data = Request.from_values( input_stream=BytesIO(data), content_length=len(data), content_type='multipart/form-data; boundary=foo', method='POST') assert not data.files strict_eq(data.form['foo'], u'a string')
def test_bad_newline_bad_newline_assumption(self): class ISORequest(Request): charset = "latin1" contents = b"U2vlbmUgbORu" data = (b'--foo\r\nContent-Disposition: form-data; name="test"\r\n' b"Content-Transfer-Encoding: base64\r\n\r\n" + contents + b"\r\n--foo--") req = ISORequest.from_values( input_stream=BytesIO(data), content_length=len(data), content_type="multipart/form-data; boundary=foo", method="POST", ) strict_eq(req.form["test"], u"Sk\xe5ne l\xe4n")
def test_end_of_file(self): # This test looks innocent but it was actually timeing out in # the Werkzeug 0.5 release version (#394) data = ( b'--foo\r\n' b'Content-Disposition: form-data; name="test"; filename="test.txt"\r\n' b'Content-Type: text/plain\r\n\r\n' b'file contents and no end') data = Request.from_values( input_stream=BytesIO(data), content_length=len(data), content_type='multipart/form-data; boundary=foo', method='POST') assert not data.files assert not data.form
def test_file_rfc2231_filename_continuations(self): data = (b"--foo\r\n" b"Content-Type: text/plain; charset=utf-8\r\n" b'Content-Disposition: form-data; name=rfc2231;\r\n' b" filename*0*=ascii''a%20b%20;\r\n" b" filename*1*=c%20d%20;\r\n" b' filename*2="e f.txt"\r\n\r\n' b"file contents\r\n--foo--") request = Request.from_values( input_stream=BytesIO(data), content_length=len(data), content_type="multipart/form-data; boundary=foo", method="POST") assert request.files["rfc2231"].filename == "a b c d e f.txt" assert request.files["rfc2231"].read() == b"file contents"
def test_headers(self): data = (b'--foo\r\n' b'Content-Disposition: form-data; name="foo"; filename="foo.txt"\r\n' b'X-Custom-Header: blah\r\n' b'Content-Type: text/plain; charset=utf-8\r\n\r\n' b'file contents, just the contents\r\n' b'--foo--') req = Request.from_values(input_stream=BytesIO(data), content_length=len(data), content_type='multipart/form-data; boundary=foo', method='POST') foo = req.files['foo'] self.assert_strict_equal(foo.mimetype, 'text/plain') self.assert_strict_equal(foo.mimetype_params, {'charset': 'utf-8'}) self.assert_strict_equal(foo.headers['content-type'], foo.content_type) self.assert_strict_equal(foo.content_type, 'text/plain; charset=utf-8') self.assert_strict_equal(foo.headers['x-custom-header'], 'blah')
def test_headers(self): data = ( b"--foo\r\n" b'Content-Disposition: form-data; name="foo"; filename="foo.txt"\r\n' b"X-Custom-Header: blah\r\n" b"Content-Type: text/plain; charset=utf-8\r\n\r\n" b"file contents, just the contents\r\n" b"--foo--") req = Request.from_values( input_stream=BytesIO(data), content_length=len(data), content_type="multipart/form-data; boundary=foo", method="POST", ) foo = req.files["foo"] strict_eq(foo.mimetype, "text/plain") strict_eq(foo.mimetype_params, {"charset": "utf-8"}) strict_eq(foo.headers["content-type"], foo.content_type) strict_eq(foo.content_type, "text/plain; charset=utf-8") strict_eq(foo.headers["x-custom-header"], "blah")
def test_streaming_parse(self): data = b'x' * (1024 * 600) class StreamMPP(formparser.MultiPartParser): def parse(self, file, boundary, content_length): i = iter( self.parse_lines(file, boundary, content_length, cap_at_buffer=False)) one = next(i) two = next(i) return self.cls(()), {'one': one, 'two': two} class StreamFDP(formparser.FormDataParser): def _sf_parse_multipart(self, stream, mimetype, content_length, options): form, files = StreamMPP( self.stream_factory, self.charset, self.errors, max_form_memory_size=self.max_form_memory_size, cls=self.cls).parse( stream, options.get('boundary').encode('ascii'), content_length) return stream, form, files parse_functions = {} parse_functions.update(formparser.FormDataParser.parse_functions) parse_functions['multipart/form-data'] = _sf_parse_multipart class StreamReq(Request): form_data_parser_class = StreamFDP req = StreamReq.from_values(data={'foo': (BytesIO(data), 'test.txt')}, method='POST') strict_eq('begin_file', req.files['one'][0]) strict_eq(('foo', 'test.txt'), req.files['one'][1][1:]) strict_eq('cont', req.files['two'][0]) strict_eq(data, req.files['two'][1])
def test_streaming_parse(self): data = b"x" * (1024 * 600) class StreamMPP(formparser.MultiPartParser): def parse(self, file, boundary, content_length): i = iter( self.parse_lines(file, boundary, content_length, cap_at_buffer=False)) one = next(i) two = next(i) return self.cls(()), {"one": one, "two": two} class StreamFDP(formparser.FormDataParser): def _sf_parse_multipart(self, stream, mimetype, content_length, options): form, files = StreamMPP( self.stream_factory, self.charset, self.errors, max_form_memory_size=self.max_form_memory_size, cls=self.cls, ).parse(stream, options.get("boundary").encode("ascii"), content_length) return stream, form, files parse_functions = {} parse_functions.update(formparser.FormDataParser.parse_functions) parse_functions["multipart/form-data"] = _sf_parse_multipart class StreamReq(Request): form_data_parser_class = StreamFDP req = StreamReq.from_values(data={"foo": (BytesIO(data), "test.txt")}, method="POST") strict_eq("begin_file", req.files["one"][0]) strict_eq(("foo", "test.txt"), req.files["one"][1][1:]) strict_eq("cont", req.files["two"][0]) strict_eq(data, req.files["two"][1])
def test_nonstandard_line_endings(self): for nl in b"\n", b"\r", b"\r\n": data = nl.join(( b"--foo", b"Content-Disposition: form-data; name=foo", b"", b"this is just bar", b"--foo", b"Content-Disposition: form-data; name=bar", b"", b"blafasel", b"--foo--", )) req = Request.from_values( input_stream=BytesIO(data), content_length=len(data), content_type="multipart/form-data; boundary=foo", method="POST", ) strict_eq(req.form["foo"], u"this is just bar") strict_eq(req.form["bar"], u"blafasel")
def test_default_stream_factory(self, no_spooled, size, monkeypatch): if no_spooled: monkeypatch.setattr("werkzeug.formparser.SpooledTemporaryFile", None) data = b"a,b,c\n" * size req = Request.from_values(data={"foo": (BytesIO(data), "test.txt")}, method="POST") file_storage = req.files["foo"] try: if PY2: reader = csv.reader(file_storage) else: reader = csv.reader(io.TextIOWrapper(file_storage)) # This fails if file_storage doesn't implement IOBase. # https://github.com/pallets/werkzeug/issues/1344 # https://github.com/python/cpython/pull/3249 assert sum(1 for _ in reader) == size finally: file_storage.close()
def parse(self, data, media_type, **options): multipart_parser = WerkzeugMultiPartParser(default_stream_factory) # TODO: dirty code boundary = media_type.split('boundary=')[-1].strip() #boundary = media_type.params.get('boundary') if boundary is None: msg = 'Multipart message missing boundary in Content-Type header' raise exceptions.ParseError(msg) boundary = boundary.encode('ascii') content_length = options.get('content_length') assert content_length is not None, 'MultiPartParser.parse() requires `content_length` argument' if data.rstrip()[-2:] != '--': data = data.rstrip() + '--\r\n' try: return multipart_parser.parse(BytesIO(data), boundary, len(data)) except ValueError as exc: msg = 'Multipart parse error - %s' % text_type(exc) raise exceptions.ParseError(msg)
def getgeojson(): if request.method == 'POST': print(request.args.to_dict()) # formdan dosya gelip gelmediğini kontrol edelim if 'file' not in request.files: print("Dosya yüklenmedi") #app.make_response(('Dosya yüklenmedi', 403)) return "Dosya yüklenmedi", 400, {'ContentType': 'text/html'} # kullanıcı dosya seçmemiş ve tarayıcı boş isim göndermiş mi file = request.files.get('file') precision = request.args.get('precision') if file.filename == '': app.make_response(('Dosya yüklenmedi', 403)) return "Dosya yüklenmedi", 403, {'ContentType': 'text/html'} # gelen dosyayı güvenlik önlemlerinden geçir if file and checkExtension(file.filename): filename = secure_filename(file.filename) #memoryshape = (file.getvalue()) zipped = filename zf = zipfile.ZipFile(file) file_like_object = file.stream._file zipfile_ob = zipfile.ZipFile(file_like_object) print(zipfile_ob.namelist()) shpname, shxname, dbfname, prjname = zipfile_ob.namelist() cloudshp = StringIO(zipfile_ob.read(shpname)) cloudshx = StringIO(zipfile_ob.read(shxname)) clouddbf = StringIO(zipfile_ob.read(dbfname)) r = shapefile.Reader(shp=cloudshp, shx=cloudshx, dbf=clouddbf) print(r.bbox) file_names = zipfile_ob.namelist() print(file_names) for item in zipfile_ob.filelist: pass # print(item) """ with open(zipfile_ob, 'rb') as file_data: bytes_content = file_data.read() """ new_zip = BytesIO() with zipfile.ZipFile(new_zip, 'w') as new_archive: for item in zipfile_ob.filelist: # If you spot an existing file, create a new object new_archive.writestr(item, zipfile_ob.read(item.filename)) """ with open(zipped, 'rb') as file_data: bytes_content = file_data.read() """ #reader = import_data(file.read()) reader = import_data(file.getvalue()) fields = reader.fields[1:] field_names = [field[0] for field in fields] print(field_names) buffer = [] for sr in reader.shapeRecords(): atr = dict(zip(field_names, sr.record)) geom = sr.shape.__geo_interface__ buffer.append(dict(type="Feature", geometry=geom, properties=atr)) feature_collection = {"type": "FeatureCollection", "features": buffer} return jsonify(feature_collection), 200, {'ContentType': 'application/json'} else: return "Hata", 500 else: abort(401)
def handle_request(application, event, context): if u"multiValueHeaders" in event: headers = Headers(event["multiValueHeaders"]) else: headers = Headers(event["headers"]) strip_stage_path = os.environ.get("STRIP_STAGE_PATH", "").lower().strip() in [ "yes", "y", "true", "t", "1", ] if u"apigw.tencentcs.com" in headers.get(u"Host", u"") and not strip_stage_path: script_name = "/{}".format(event["requestContext"].get(u"stage", "")) else: script_name = "" path_info = event["path"] base_path = os.environ.get("API_GATEWAY_BASE_PATH") if base_path: script_name = "/" + base_path if path_info.startswith(script_name): path_info = path_info[len(script_name):] or "/" if u"body" in event: body = event[u"body"] or "" else: body = "" if event.get("isBase64Encoded", False): body = base64.b64decode(body) if isinstance(body, string_types): body = to_bytes(body, charset="utf-8") environ = { "CONTENT_LENGTH": str(len(body)), "CONTENT_TYPE": headers.get(u"Content-Type", ""), "PATH_INFO": url_unquote(path_info), "QUERY_STRING": encode_query_string(event), "REMOTE_ADDR": event["requestContext"].get(u"identity", {}).get(u"sourceIp", ""), "REMOTE_USER": event["requestContext"].get(u"authorizer", {}).get(u"principalId", ""), "REQUEST_METHOD": event["httpMethod"], "SCRIPT_NAME": script_name, "SERVER_NAME": headers.get(u"Host", "lambda"), "SERVER_PORT": headers.get(u"X-Forwarded-Port", "80"), "SERVER_PROTOCOL": "HTTP/1.1", "wsgi.errors": sys.stderr, "wsgi.input": BytesIO(body), "wsgi.multiprocess": False, "wsgi.multithread": False, "wsgi.run_once": False, "wsgi.url_scheme": headers.get(u"X-Forwarded-Proto", "http"), "wsgi.version": (1, 0), "serverless.authorizer": event["requestContext"].get(u"authorizer"), "serverless.event": event, "serverless.context": context, # TODO: Deprecate the following entries, as they do not comply with the WSGI # spec. For custom variables, the spec says: # # Finally, the environ dictionary may also contain server-defined variables. # These variables should be named using only lower-case letters, numbers, dots, # and underscores, and should be prefixed with a name that is unique to the # defining server or gateway. "API_GATEWAY_AUTHORIZER": event["requestContext"].get(u"authorizer"), "event": event, "context": context, } for key, value in environ.items(): if isinstance(value, string_types): environ[key] = wsgi_encoding_dance(value) for key, value in headers.items(): key = "HTTP_" + key.upper().replace("-", "_") if key not in ("HTTP_CONTENT_TYPE", "HTTP_CONTENT_LENGTH"): environ[key] = value response = Response.from_app(application, environ) returndict = {u"statusCode": response.status_code} if u"multiValueHeaders" in event: returndict["multiValueHeaders"] = group_headers(response.headers) else: returndict["headers"] = split_headers(response.headers) if event.get("requestContext").get("elb"): # If the request comes from ALB we need to add a status description returndict["statusDescription"] = u"%d %s" % ( response.status_code, HTTP_STATUS_CODES[response.status_code], ) if response.data: mimetype = response.mimetype or "text/plain" if (mimetype.startswith("text/") or mimetype in TEXT_MIME_TYPES) and not response.headers.get( "Content-Encoding", ""): returndict["body"] = response.get_data(as_text=True) returndict["isBase64Encoded"] = False else: returndict["body"] = base64.b64encode( response.data).decode("utf-8") returndict["isBase64Encoded"] = True return returndict