def kv_session(self, index, key, value=None): """Set up connection context for admin KV store API.""" canonical_uri = '/indexes/{}/{}'.format( urllib.parse.quote(index, safe=""), urllib.parse.quote(key)) request_uri = self._session.admin_endpoint + canonical_uri query_params = "" body = value or "" headers = AWSV4Signer(self._session.admin_endpoint, self._session.service_name, self._session.region, self._session.access_key, self._session.secret_key).prepare_signed_header( 'GET' if value is None else 'PUT', canonical_uri, query_params, body) if (headers['Authorization'] is None): self._logger.error( fmt_reqid_log(self._request_id) + "Failed to generate v4 signature") sys.exit(-1) self._logger.info( fmt_reqid_log(self._request_id) + 'Motr index operation on {} {}'.format(request_uri, body)) if value is None: # Called without a new value, assumed to be an HTTP GET return self._session.get_client_session().get(URL(request_uri, encoded=True), params=query_params, headers=headers) else: # Going to PUT the new value return self._session.get_client_session().put(URL(request_uri, encoded=True), params=query_params, headers=headers, data=body.encode())
async def upload(self, data_reader, part_no, chunk_size): self._state = S3RequestState.RUNNING self._part_no = part_no request_uri = AWSV4Signer.fmt_s3_request_uri(self._bucket_name, self._object_name) print("Part Number : {}".format(self._part_no)) query_params = urllib.parse.urlencode({ 'partNumber': self._part_no, 'uploadId': self._upload_id }) body = "" headers = AWSV4Signer(self._session.endpoint, self._session.service_name, self._session.region, self._session.access_key, self._session.secret_key).prepare_signed_header( 'PUT', request_uri, query_params, body) if (headers['Authorization'] is None): self._logger.error( fmt_reqid_log(self._request_id) + "Failed to generate v4 signature") sys.exit(-1) headers["Content-Length"] = str(chunk_size) self._logger.info( fmt_reqid_log(self._request_id) + "PUT on {}".format(self._session.endpoint + request_uri)) self._logger.debug( fmt_reqid_log(self._request_id) + "PUT with headers {}".format(headers)) self._timer.start() try: async with self._session.get_client_session().put( self._session.endpoint + request_uri, headers=headers, params=query_params, data=data_reader.fetch(chunk_size)) as resp: self._timer.stop() self._http_status = resp.status self._response_headers = resp.headers self._logger.info( fmt_reqid_log(self._request_id) + 'PUT Object completed with http status: {}' '\n header{}'.format(resp.status, self._response_headers)) self._etag_dict[self._part_no] = self._response_headers["Etag"] if resp.status == 200: self._state = S3RequestState.COMPLETED else: error_msg = await resp.text() self._logger.error( fmt_reqid_log(self._request_id) + 'Error Response: {}'.format(error_msg)) self._state = S3RequestState.FAILED except aiohttp.client_exceptions.ClientConnectorError as e: self._timer.stop() self.remote_down = True self._state = S3RequestState.FAILED self._logger.error( fmt_reqid_log(self._request_id) + "Failed to connect to S3: " + str(e)) return
async def get(self): """Yields data chunk for given size.""" request_uri = AWSV4Signer.fmt_s3_request_uri(self._bucket_name) self._logger.debug( fmt_reqid_log(self._request_id) + "request_uri : {}".format(request_uri)) query_params = urllib.parse.urlencode({'replication': None}) body = "" headers = AWSV4Signer(self._session.endpoint, self._session.service_name, self._session.region, self._session.access_key, self._session.secret_key).prepare_signed_header( 'GET', request_uri, query_params, body) if (headers['Authorization'] is None): self._logger.error( fmt_reqid_log(self._request_id) + "Failed to generate v4 signature") sys.exit(-1) # Request url url = self._session.endpoint + request_uri self._logger.info( fmt_reqid_log(self._request_id) + 'GET on {}'.format(url)) self._timer.start() try: async with self._session.get_client_session().get( url, params=query_params, headers=headers) as resp: self._logger.debug( fmt_reqid_log(self._request_id) + "Response url {}".format((resp.url))) self._logger.debug( fmt_reqid_log(self._request_id) + "Received response url {}".format(resp)) if resp.status == 200: self._logger.info( fmt_reqid_log(self._request_id) + "Received reponse [{} OK]".format(resp.status)) xml_resp = await resp.text() self._response_dict = xmltodict.parse(xml_resp) self._logger.debug('Response xml : {}\n'.format( self._response_dict)) else: self._state = S3RequestState.FAILED error_msg = await resp.text() self._logger.error( fmt_reqid_log(self._request_id) + 'Error Response: {}'.format(error_msg)) except Exception as e: self._logger.error( fmt_reqid_log(self._request_id) + "Error: Exception '{}' occured!".format(e)) self._timer.stop() self._logger.debug( fmt_reqid_log(self._request_id) + "execution time is : {}".format(self.get_execution_time())) return
async def send(self, data_reader, transfer_size): self._state = S3RequestState.RUNNING self._data_reader = data_reader request_uri = AWSV4Signer.fmt_s3_request_uri(self._bucket_name, self._object_name) query_params = "" body = "" headers = AWSV4Signer(self._session.endpoint, self._session.service_name, self._session.region, self._session.access_key, self._session.secret_key).prepare_signed_header( 'PUT', request_uri, query_params, body) if (headers['Authorization'] is None): self._logger.error( fmt_reqid_log(self._request_id) + "Failed to generate v4 signature") sys.exit(-1) headers["Content-Length"] = str(self._object_size) self._logger.info( fmt_reqid_log(self._request_id) + "PUT on {}".format(self._session.endpoint + request_uri)) self._logger.debug( fmt_reqid_log(self._request_id) + "PUT with headers {}".format(headers)) self._timer.start() try: async with self._session.get_client_session().put( self._session.endpoint + request_uri, headers=headers, # Read all data from data_reader data=data_reader.fetch(transfer_size)) as resp: self._timer.stop() if data_reader.get_state() != S3RequestState.ABORTED: self._http_status = resp.status self._response_headers = resp.headers self._logger.info( fmt_reqid_log(self._request_id) + 'PUT Object completed with http status: {}'.format( resp.status)) # Validate if upload object etag matches. if self.get_etag() != data_reader.get_etag(): self._state = S3RequestState.FAILED error_msg = "ETag mismatch." self._logger.error( fmt_reqid_log(self._request_id) + 'Error Response: {}'.format(error_msg)) if resp.status == 200: self._state = S3RequestState.COMPLETED else: error_msg = await resp.text() self._logger.error( fmt_reqid_log(self._request_id) + 'Error Response: {}'.format(error_msg)) self._state = S3RequestState.FAILED except aiohttp.client_exceptions.ClientConnectorError as e: self._timer.stop() self.remote_down = True self._state = S3RequestState.FAILED self._logger.error( fmt_reqid_log(self._request_id) + "Failed to connect to S3: " + str(e)) return