def describe_dump_task(self,
streamName,
task_name,
ak="",
sk="",
xSecrityToken=""):
self.__assert_to_validate(streamName, "the stream Name is null")
self.__assert_to_validate(task_name, "the task_name is null")
uri = "/v2/" + self.projectid + "/stream/" + streamName + "/" + "transfer-tasks/"
req = self._generateRequest("GET",
uri,
headers={},
body="",
userak=ak,
usersk=sk,
userxSecrityToken=xSecrityToken)
(statusCode, responseData) = self._sendRequest(req)
for i in responseData['details']:
if task_name == i['task_name']:
return disadminresponse.disdescribedumptaskResponse(
statusCode, i)
else:
raise DisException('errormeaasge',
'%s not exists' % (i['task_name']))
def describeStream(self,
streamName,
startPartitionId="",
limitPartitions=1000,
ak="",
sk="",
xSecrityToken=""):
self.__assert_to_validate(streamName, "the streamname is null")
param = {}
if startPartitionId is not '':
param["start_partitionId"] = startPartitionId
if limitPartitions > 10000:
raise DisException("invalidparam", "the limit is to large")
if limitPartitions != 0:
param["limit_partitions"] = str(limitPartitions)
uri = "/v2/" + self.projectid + "/streams/" + streamName + "/"
req = self._generateRequest("GET",
uri,
query=param,
headers={},
body="",
userak=ak,
usersk=sk,
userxSecrityToken=xSecrityToken)
(statusCode, responseData) = self._sendRequest(req)
return disadminresponse.disDescribeStreamresultResponse(
statusCode, responseData)
def _getResponse(self,
method,
uri,
timeout,
params={},
headers={},
body=""):
try:
r = requests.request(method,
uri,
params=params,
data=body,
headers=headers,
timeout=timeout,
verify=False)
return r.status_code, r.content, r.headers
except Exception as ex:
raise DisException('_getResponse', str(ex))
def Applist(self, appName, ak="", sk="", xSecrityToken="", limit=100):
param = {}
uri = "/v2/" + self.projectid + "/apps"
req = self._generateRequest("GET",
uri,
headers={},
query={'limit': '100'},
userak=ak,
usersk=sk,
userxSecrityToken=xSecrityToken)
(statusCode, responseData) = self._sendRequest(req)
if responseData.get('apps'):
total_number = len(responseData.get('apps'))
else:
total_number = ''
if limit > 100:
raise DisException("invalidparam", "the limit is to large")
if limit != 0:
param["limit"] = str(limit)
if appName is not '':
self.__assert_to_validate(appName, "the appname is null")
param["start_app_name"] = appName
uri = "/v2/" + self.projectid + "/apps"
req = self._generateRequest("GET",
uri,
headers={},
query=param,
userak=ak,
usersk=sk,
userxSecrityToken=xSecrityToken)
(statusCode, responseData) = self._sendRequest(req)
return discheckpointresponse.disApplistResponse(
statusCode, responseData, total_number)
def listStream(self,
startStreamName="",
limit=100,
ak="",
sk="",
xSecrityToken=""):
'''
list all of the stream of the user.
the MAX of limit is 100,if larger than 100, the sdk should raise exception
'''
param = {}
if startStreamName is not '':
self.__assert_to_validate(startStreamName,
"the stream Name is null")
param["start_stream_name"] = startStreamName
if limit > 100:
raise DisException("invalidparam", "the limit is to large")
if limit != 0:
param["limit"] = str(limit)
uri = "/v2/" + self.projectid + "/streams/"
req = self._generateRequest("GET",
uri,
query=param,
headers={},
body="",
userak=ak,
usersk=sk,
userxSecrityToken=xSecrityToken)
(statusCode, responseData) = self._sendRequest(req)
return disadminresponse.disListStreamResponse(statusCode, responseData)
def getCursor(self,
streamName,
partitionId,
cursorType,
startSeq,
ak="",
sk="",
xSecrityToken=""):
'''
the cursor is the pointer to get the data in partition.
:type streamName string
:param streamName the streamName ID which want to send data
:type partitionId string
:param partitionId the partition ID which want to get data, you can get all of the partition info from describeStream interface
:type cursorType string
:param cursorType. there are four type for the cursor
:AT_SEQUENCE_NUMBER The consumer application starts reading from the position denoted by a specific sequence number. This is the default Cursor Type.
:AFTER_SEQUENCE_NUMBER The consumer application starts reading right after the position denoted by a specific sequence number.
:TRIM_HORIZON The consumer application starts reading at the last untrimmed record in the partition in the system, which is the oldest data record in the partition.
:LATEST Start reading just after the most recent record in the partition, so that you always read the most recent data in the partition
:type startSeq tring
:param startSeq
Sequence number of the data record in the partition from which to start reading.
Value range: 0 to 9223372036854775807
Each data record has a sequence number that is unique within its partition. The sequence number is assigned by DIS when a data producer calls PutRecords to add data to a DIS stream.
Sequence numbers for the same partition key generally increase over time; the longer the time period between write requests (PutRecords requests), the larger the sequence numbers become.
'''
param = {}
self.__assert_to_validate(streamName, "the streamname is null")
param["stream-name"] = streamName
param["partition-id"] = partitionId
if (cursorType != "AT_SEQUENCE_NUMBER"
and cursorType != "AFTER_SEQUENCE_NUMBER"
and cursorType != "TRIM_HORIZON" and cursorType != "LATEST"):
raise DisException("Invalid param", "the cursor type is invalid")
param["cursor-type"] = cursorType
if (cursorType == "AT_SEQUENCE_NUMBER"
or cursorType == "AFTER_SEQUENCE_NUMBER"):
param["starting-sequence-number"] = startSeq
uri = "/v2/" + self.projectid + "/cursors/"
req = self._generateRequest("GET",
uri,
query=param,
headers={},
body="",
userak=ak,
usersk=sk,
userxSecrityToken=xSecrityToken)
(statusCode, responseData) = self._sendRequest(req)
return disrecordresponse.disGetCursorResponse(statusCode, responseData)
def _sendRequest(self, rawRequest):
retryCount = 0
url = self.endpoint + rawRequest.uri
wait = 0.05
while retryCount <= EXCEPTION_RETRIES:
try:
if retryCount != 0:
time.sleep(wait)
wait = wait * 2
rawRequest.headers.pop(disauth.HeaderXDate)
rawRequest.headers.pop(disauth.HeaderAuthorization)
disauth.Signer(self.ak, self.sk,
self.region).Sign(rawRequest)
r = requests.request(method=rawRequest.method,
url=url,
params=rawRequest.query,
data=rawRequest.body,
headers=rawRequest.headers,
timeout=self.TIME_OUT,
verify=False)
if r.status_code >= 200 and r.status_code < 300:
if r.content is not b'':
if 'protobuf' in rawRequest.headers["Content-Type"]:
jsonResponse = r.content
else:
jsonResponse = r.json()
if jsonResponse:
return r.status_code, jsonResponse
else:
return r.status_code, {}
else:
return r.status_code, {}
else:
errMsg = ''
errNo = ''
if r._content is not b'':
errNo = str(r.status_code)
errMsg = str(r._content)
#if r.status_code >= 500:
# try:
# jsonResponse = r.json()
# errMsg = jsonResponse["message"]
# errNo = jsonResponse["errorCode"]
# except:pass
raise DisException("GetResponseErr", "the response is err",
r.status_code, errNo, errMsg)
except Exception as ex:
if retryCount < EXCEPTION_RETRIES and \
(type(ex) == DisException and ex.respStatus >= 500
or "connect timeout" in str(ex)
or ("read timeout" in str(ex) and rawRequest.method == "GET")):
log("Find Retriable Exception [" + str(ex) + "], url [" +
rawRequest.method + " " + rawRequest.uri +
"], currRetryCount is " + str(retryCount))
retryCount = retryCount + 1
else:
if type(ex) == DisException:
raise ex
else:
raise DisException('_sendRequest', str(ex))