class S3Bounce(OFSInterface):
"""
Use ckanext-storage API to bounce to an S3 store
"""
def __init__(self, api_base):
self.ckan = CkanClient(base_location=api_base)
def put_stream(self, bucket, label, fp, metadata={}, cb=None, num_cb=None):
if metadata is None:
metadata = { "_owner": getpass.getuser()}
path = "/" + bucket + "/" + label
content_type = metadata.get("_format", "application/octet-stream")
metadata = self.ckan.storage_metadata_set(path, metadata)
BufferSize = 65536 ## set to something very small to make sure
## chunking is working properly
headers = { 'Content-Type': content_type }
#if content_type is None:
# content_type = mimetypes.guess_type(filename)[0] or "text/plain"
#headers['Content-Type'] = content_type
#if content_encoding is not None:
# headers['Content-Encoding'] = content_encoding
m = md5()
fp.seek(0)
s = fp.read(BufferSize)
while s:
m.update(s)
s = fp.read(BufferSize)
self.size = fp.tell()
fp.seek(0)
self.md5 = m.hexdigest()
headers['Content-MD5'] = base64.encodestring(m.digest()).rstrip('\n')
headers['Content-Length'] = str(self.size)
headers['Expect'] = '100-Continue'
host, headers = self.ckan.storage_auth_get(path, headers)
def sender(http_conn, method, path, data, headers):
http_conn.putrequest(method, path)
for key in headers:
http_conn.putheader(key, headers[key])
http_conn.endheaders()
fp.seek(0)
http_conn.set_debuglevel(0) ### XXX set to e.g. 4 to see what going on
if cb:
if num_cb > 2:
cb_count = self.size / BufferSize / (num_cb-2)
elif num_cb < 0:
cb_count = -1
else:
cb_count = 0
i = total_bytes = 0
cb(total_bytes, self.size)
l = fp.read(BufferSize)
while len(l) > 0:
http_conn.send(l)
if cb:
total_bytes += len(l)
i += 1
if i == cb_count or cb_count == -1:
cb(total_bytes, self.size)
i = 0
l = fp.read(BufferSize)
if cb:
cb(total_bytes, self.size)
response = http_conn.getresponse()
body = response.read()
fp.seek(0)
if response.status == 500 or response.status == 503 or \
response.getheader('location'):
# we'll try again
return response
elif response.status >= 200 and response.status <= 299:
self.etag = response.getheader('etag')
if self.etag != '"%s"' % self.md5:
raise Exception('ETag from S3 did not match computed MD5')
return response
else:
#raise provider.storage_response_error(
# response.status, response.reason, body)
raise Exception(response.status, response.reason, body)
awsc = AWSAuthConnection(host,
aws_access_key_id="key_id",
aws_secret_access_key="secret")
awsc._mexe('PUT', path, None, headers, sender=sender)
metadata = self.ckan.storage_metadata_update(path, {})
from pprint import pprint
pprint(metadata)
class S3Bounce(OFSInterface):
"""
Use ckanext-storage API to bounce to an S3 store
"""
def __init__(self, api_base):
self.ckan = CkanClient(base_location=api_base)
def put_stream(self, bucket, label, fp, metadata={}, cb=None, num_cb=None):
if metadata is None:
metadata = {"_owner": getpass.getuser()}
path = "/" + bucket + "/" + label
content_type = metadata.get("_format", "application/octet-stream")
metadata = self.ckan.storage_metadata_set(path, metadata)
BufferSize = 65536 ## set to something very small to make sure
## chunking is working properly
headers = {'Content-Type': content_type}
#if content_type is None:
# content_type = mimetypes.guess_type(filename)[0] or "text/plain"
#headers['Content-Type'] = content_type
#if content_encoding is not None:
# headers['Content-Encoding'] = content_encoding
m = md5()
fp.seek(0)
s = fp.read(BufferSize)
while s:
m.update(s)
s = fp.read(BufferSize)
self.size = fp.tell()
fp.seek(0)
self.md5 = m.hexdigest()
headers['Content-MD5'] = base64.encodestring(m.digest()).rstrip('\n')
headers['Content-Length'] = str(self.size)
headers['Expect'] = '100-Continue'
host, headers = self.ckan.storage_auth_get(path, headers)
def sender(http_conn, method, path, data, headers):
http_conn.putrequest(method, path)
for key in headers:
http_conn.putheader(key, headers[key])
http_conn.endheaders()
fp.seek(0)
http_conn.set_debuglevel(
0) ### XXX set to e.g. 4 to see what going on
if cb:
if num_cb > 2:
cb_count = self.size / BufferSize / (num_cb - 2)
elif num_cb < 0:
cb_count = -1
else:
cb_count = 0
i = total_bytes = 0
cb(total_bytes, self.size)
l = fp.read(BufferSize)
while len(l) > 0:
http_conn.send(l)
if cb:
total_bytes += len(l)
i += 1
if i == cb_count or cb_count == -1:
cb(total_bytes, self.size)
i = 0
l = fp.read(BufferSize)
if cb:
cb(total_bytes, self.size)
response = http_conn.getresponse()
body = response.read()
fp.seek(0)
if response.status == 500 or response.status == 503 or \
response.getheader('location'):
# we'll try again
return response
elif response.status >= 200 and response.status <= 299:
self.etag = response.getheader('etag')
if self.etag != '"%s"' % self.md5:
raise Exception('ETag from S3 did not match computed MD5')
return response
else:
#raise provider.storage_response_error(
# response.status, response.reason, body)
raise Exception(response.status, response.reason, body)
awsc = AWSAuthConnection(host,
aws_access_key_id="key_id",
aws_secret_access_key="secret")
awsc._mexe('PUT', path, None, headers, sender=sender)
metadata = self.ckan.storage_metadata_update(path, {})
from pprint import pprint
pprint(metadata)
