def kv_session(self, index, key, value=None):
"""Set up connection context for admin KV store API."""
canonical_uri = '/indexes/{}/{}'.format(
urllib.parse.quote(index, safe=""), urllib.parse.quote(key))
request_uri = self._session.admin_endpoint + canonical_uri
query_params = ""
body = value or ""
headers = AWSV4Signer(self._session.admin_endpoint,
self._session.service_name, self._session.region,
self._session.access_key,
self._session.secret_key).prepare_signed_header(
'GET' if value is None else 'PUT',
canonical_uri, query_params, body)
if (headers['Authorization'] is None):
self._logger.error(
fmt_reqid_log(self._request_id) +
"Failed to generate v4 signature")
sys.exit(-1)
self._logger.info(
fmt_reqid_log(self._request_id) +
'Motr index operation on {} {}'.format(request_uri, body))
if value is None:
# Called without a new value, assumed to be an HTTP GET
return self._session.get_client_session().get(URL(request_uri,
encoded=True),
params=query_params,
headers=headers)
else:
# Going to PUT the new value
return self._session.get_client_session().put(URL(request_uri,
encoded=True),
params=query_params,
headers=headers,
data=body.encode())
async def upload(self, data_reader, part_no, chunk_size):
self._state = S3RequestState.RUNNING
self._part_no = part_no
request_uri = AWSV4Signer.fmt_s3_request_uri(self._bucket_name,
self._object_name)
print("Part Number : {}".format(self._part_no))
query_params = urllib.parse.urlencode({
'partNumber': self._part_no,
'uploadId': self._upload_id
})
body = ""
headers = AWSV4Signer(self._session.endpoint,
self._session.service_name, self._session.region,
self._session.access_key,
self._session.secret_key).prepare_signed_header(
'PUT', request_uri, query_params, body)
if (headers['Authorization'] is None):
self._logger.error(
fmt_reqid_log(self._request_id) +
"Failed to generate v4 signature")
sys.exit(-1)
headers["Content-Length"] = str(chunk_size)
self._logger.info(
fmt_reqid_log(self._request_id) +
"PUT on {}".format(self._session.endpoint + request_uri))
self._logger.debug(
fmt_reqid_log(self._request_id) +
"PUT with headers {}".format(headers))
self._timer.start()
try:
async with self._session.get_client_session().put(
self._session.endpoint + request_uri,
headers=headers,
params=query_params,
data=data_reader.fetch(chunk_size)) as resp:
self._timer.stop()
self._http_status = resp.status
self._response_headers = resp.headers
self._logger.info(
fmt_reqid_log(self._request_id) +
'PUT Object completed with http status: {}'
'\n header{}'.format(resp.status, self._response_headers))
self._etag_dict[self._part_no] = self._response_headers["Etag"]
if resp.status == 200:
self._state = S3RequestState.COMPLETED
else:
error_msg = await resp.text()
self._logger.error(
fmt_reqid_log(self._request_id) +
'Error Response: {}'.format(error_msg))
self._state = S3RequestState.FAILED
except aiohttp.client_exceptions.ClientConnectorError as e:
self._timer.stop()
self.remote_down = True
self._state = S3RequestState.FAILED
self._logger.error(
fmt_reqid_log(self._request_id) + "Failed to connect to S3: " +
str(e))
return
async def get(self):
"""Yields data chunk for given size."""
request_uri = AWSV4Signer.fmt_s3_request_uri(self._bucket_name)
self._logger.debug(
fmt_reqid_log(self._request_id) +
"request_uri : {}".format(request_uri))
query_params = urllib.parse.urlencode({'replication': None})
body = ""
headers = AWSV4Signer(self._session.endpoint,
self._session.service_name, self._session.region,
self._session.access_key,
self._session.secret_key).prepare_signed_header(
'GET', request_uri, query_params, body)
if (headers['Authorization'] is None):
self._logger.error(
fmt_reqid_log(self._request_id) +
"Failed to generate v4 signature")
sys.exit(-1)
# Request url
url = self._session.endpoint + request_uri
self._logger.info(
fmt_reqid_log(self._request_id) + 'GET on {}'.format(url))
self._timer.start()
try:
async with self._session.get_client_session().get(
url, params=query_params, headers=headers) as resp:
self._logger.debug(
fmt_reqid_log(self._request_id) +
"Response url {}".format((resp.url)))
self._logger.debug(
fmt_reqid_log(self._request_id) +
"Received response url {}".format(resp))
if resp.status == 200:
self._logger.info(
fmt_reqid_log(self._request_id) +
"Received reponse [{} OK]".format(resp.status))
xml_resp = await resp.text()
self._response_dict = xmltodict.parse(xml_resp)
self._logger.debug('Response xml : {}\n'.format(
self._response_dict))
else:
self._state = S3RequestState.FAILED
error_msg = await resp.text()
self._logger.error(
fmt_reqid_log(self._request_id) +
'Error Response: {}'.format(error_msg))
except Exception as e:
self._logger.error(
fmt_reqid_log(self._request_id) +
"Error: Exception '{}' occured!".format(e))
self._timer.stop()
self._logger.debug(
fmt_reqid_log(self._request_id) +
"execution time is : {}".format(self.get_execution_time()))
return
async def send(self, data_reader, transfer_size):
self._state = S3RequestState.RUNNING
self._data_reader = data_reader
request_uri = AWSV4Signer.fmt_s3_request_uri(self._bucket_name,
self._object_name)
query_params = ""
body = ""
headers = AWSV4Signer(self._session.endpoint,
self._session.service_name, self._session.region,
self._session.access_key,
self._session.secret_key).prepare_signed_header(
'PUT', request_uri, query_params, body)
if (headers['Authorization'] is None):
self._logger.error(
fmt_reqid_log(self._request_id) +
"Failed to generate v4 signature")
sys.exit(-1)
headers["Content-Length"] = str(self._object_size)
self._logger.info(
fmt_reqid_log(self._request_id) +
"PUT on {}".format(self._session.endpoint + request_uri))
self._logger.debug(
fmt_reqid_log(self._request_id) +
"PUT with headers {}".format(headers))
self._timer.start()
try:
async with self._session.get_client_session().put(
self._session.endpoint + request_uri,
headers=headers,
# Read all data from data_reader
data=data_reader.fetch(transfer_size)) as resp:
self._timer.stop()
if data_reader.get_state() != S3RequestState.ABORTED:
self._http_status = resp.status
self._response_headers = resp.headers
self._logger.info(
fmt_reqid_log(self._request_id) +
'PUT Object completed with http status: {}'.format(
resp.status))
# Validate if upload object etag matches.
if self.get_etag() != data_reader.get_etag():
self._state = S3RequestState.FAILED
error_msg = "ETag mismatch."
self._logger.error(
fmt_reqid_log(self._request_id) +
'Error Response: {}'.format(error_msg))
if resp.status == 200:
self._state = S3RequestState.COMPLETED
else:
error_msg = await resp.text()
self._logger.error(
fmt_reqid_log(self._request_id) +
'Error Response: {}'.format(error_msg))
self._state = S3RequestState.FAILED
except aiohttp.client_exceptions.ClientConnectorError as e:
self._timer.stop()
self.remote_down = True
self._state = S3RequestState.FAILED
self._logger.error(
fmt_reqid_log(self._request_id) + "Failed to connect to S3: " +
str(e))
return