def _stream_upload_segment(self, uuid, offset, length, in_fp,
storage_metadata, content_type):
updated_metadata = copy.deepcopy(storage_metadata)
segment_count = len(updated_metadata[_SEGMENTS_KEY])
segment_path = "%s/%s/%s" % (_SEGMENT_DIRECTORY, uuid,
"%09d" % segment_count)
# Track the number of bytes read and if an explicit length is specified, limit the
# file stream to that length.
if length == filelike.READ_UNTIL_END:
length = _MAXIMUM_SEGMENT_SIZE
else:
length = min(_MAXIMUM_SEGMENT_SIZE, length)
limiting_fp = filelike.LimitingStream(in_fp, length)
# If retries are requested, then we need to use a buffered reader to allow for calls to
# seek() on retries from within the Swift client.
if self._retry_count > 0:
limiting_fp = BufferedReader(limiting_fp, buffer_size=length)
# Write the segment to Swift.
self.stream_write(segment_path, limiting_fp, content_type)
# We are only going to track keys to which data was confirmed written.
bytes_written = limiting_fp.tell()
if bytes_written > 0:
updated_metadata[_SEGMENTS_KEY].append(
_PartUploadMetadata(segment_path, offset, bytes_written))
else:
updated_metadata[_EMPTY_SEGMENTS_KEY].append(
_PartUploadMetadata(segment_path, offset, bytes_written))
return bytes_written, updated_metadata
def get(self, file_id):
path = self._files.get_file_id_path(file_id)
try:
file_stream = self._storage.stream_read_file(self._locations, path)
buffered = BufferedReader(file_stream)
file_header_bytes = buffered.peek(1024)
return send_file(
buffered,
mimetype=self._magic.from_buffer(file_header_bytes),
as_attachment=True,
attachment_filename=file_id)
except IOError:
logger.exception('Error reading user file')
abort(404)
def queryConflict(self, user, repo, layer, transactionid):
headers = {
"Accept": "application/geogig.x-protobuf",
"Accept-Encoding": "gzip",
"geogig-transaction-id": transactionid
}
query = {"conflicts": True}
url = QueryForLayerProto.createURL(self.connector, user, repo, layer)
auth_user = self.connector.user
auth_pass = self.connector.password
r = requests.post(url,
headers=headers,
json=query,
stream=True,
auth=(auth_user, auth_pass))
r.raise_for_status()
r.raw.decode_content = True
if self.isCancelled:
r.close()
raise CancelledException("cancelled")
bufferedReader = BufferedReader(r.raw, 50 * 1024)
ftHelper = FeatureTypeHelper(bufferedReader)
fiterator = self.readConflictFeatures(bufferedReader, ftHelper)
return list(fiterator)
def queryDiff(self,
user,
repo,
layer,
refspec,
oldRef,
featureFilter=None,
returnAsIterator=False,
oldRefUser=None,
oldRefRepoName=None):
headers = {
"Accept": "application/geogig.x-protobuf",
"Accept-Encoding": "gzip"
}
query = {
"head": refspec,
"flattenDiffSchema": False,
"oldHead": oldRef
}
if oldRefUser is not None and oldRefRepoName is not None:
query["oldHeadUser"] = oldRefUser
query["oldHeadRepo"] = oldRefRepoName
if featureFilter is not None:
query["filter"] = featureFilter
url = QueryForLayerProto.createURL(self.connector, user, repo, layer)
auth_user = self.connector.user
auth_pass = self.connector.password
r = requests.post(url,
headers=headers,
json=query,
stream=True,
auth=(auth_user, auth_pass))
r.raise_for_status()
r.raw.decode_content = True
if self.isCancelled:
r.close()
raise CancelledException("cancelled")
bufferedReader = BufferedReader(r.raw, 50 * 1024)
ftHelper = FeatureTypeHelper(bufferedReader)
fiterator = self.readDiffFeatures(bufferedReader, ftHelper)
if returnAsIterator:
return ftHelper, fiterator
else:
return list(fiterator)
def test_magic():
mgc = magic.Magic(mime=True)
with GeneratorFile(mimed_html_generator()) as f:
buffered = BufferedReader(f)
file_header_bytes = buffered.peek(1024)
assert mgc.from_buffer(file_header_bytes) == "text/html"
with GeneratorFile(sample_generator()) as f:
buffered = BufferedReader(f)
file_header_bytes = buffered.peek(1024)
assert mgc.from_buffer(file_header_bytes) == "text/plain"
def queryDiffPR(self, user, repo, layer, prID):
headers = {
"Accept": "application/geogig.x-protobuf",
"Accept-Encoding": "gzip"
}
url = self.createURL_PRDIFF(self.connector, user, repo, layer, prID)
auth_user = self.connector.user
auth_pass = self.connector.password
r = requests.get(url,
headers=headers,
stream=True,
auth=(auth_user, auth_pass))
r.raise_for_status()
r.raw.decode_content = True
bufferedReader = BufferedReader(r.raw, 50 * 1024)
ftHelper = FeatureTypeHelper(bufferedReader)
fiterator = self.readDiffFeatures(bufferedReader, ftHelper)
return ftHelper, fiterator
def test_with_bufferedreader():
with GeneratorFile(sample_generator()) as f:
buffered = BufferedReader(f)
assert buffered.peek(10) == "thisisatest"
assert buffered.read(10) == "thisisates"