def uploadNewFile(self, localPath, tempFilePath, parentFolderId, description = None):
if os.path.exists(localPath):
if localPath != tempFilePath:
self.logger.debug('[Adapter] Copy local file (%s) to temp file (%s)', localPath, tempFilePath)
shutil.copyfile(localPath, tempFilePath)
else:
raise WaitingOtherResourceException
sha1 = Utils.sha1ForFile(localPath)
checkExistResponse = self.autoRepeatQuery(self.getFangCloudHttpClient().checkFileExistsOnServerAtTheSameDirWithTheSameName, [localPath,
parentFolderId,
sha1,
description])
if checkExistResponse['success'] == False or not checkExistResponse['is_file_existed']:
presignResponse = self.autoRepeatQuery(self.getFangCloudHttpClient().uploadNewFilePresign, [parentFolderId, localPath, description])
if presignResponse.get('success', False):
uploadURL = presignResponse['upload_url']
response = self.autoRepeatQuery(self.getFangCloudHttpClient().uploadNewFile, [uploadURL, localPath, tempFilePath])
else:
response = presignResponse
if response['success'] == False:
result = response
else:
result = self.__formatUploadNewFileResponse(response)
return result
else:
self.logger.debug('[Adapter] File (%s) exist on server', localPath)
checkExistResponse['file_info']['success'] = checkExistResponse['success']
checkExistResponse['file_info']['is_synced'] = checkExistResponse['is_synced']
checkExistResponse['file_info']['unencrypted_sha1'] = Utils.sha1ForFile(localPath)
response = checkExistResponse['file_info']
return self.__formatUploadNewFileResponse(response)
def uploadNewFileVersion(self, localPath, tempFilePath, resourceId, sequenceId, description = 'UpdateNewVersion'):
if os.path.exists(localPath):
if localPath != tempFilePath:
shutil.copyfile(unicode(localPath), tempFilePath)
else:
raise WaitingOtherResourceException
size = os.path.getsize(localPath)
name = Utils.getNameFromPath(localPath)
modifiedAt = Utils.getModifyTime(localPath)
presignResponse = self.autoRepeatQuery(self.getFangCloudHttpClient().uploadNewFileVersionPresign, [-resourceId,
sequenceId,
size,
name,
modifiedAt])
if presignResponse.get('success', False):
uploadURL = presignResponse['upload_url']
response = self.autoRepeatQuery(self.getFangCloudHttpClient().uploadNewFileVersion, [uploadURL,
localPath,
tempFilePath,
description])
else:
response = presignResponse
if response['success']:
result = self.__formatUploadNewFileResponse(response)
else:
result = response
return result
def postLogFile(self, url, localPath, tempFile, query = None):
response = None
self.logger.debug('[HTTP] post log file, the path is %s', localPath)
httpSession = requests.Session()
try:
if query is not None:
query = self.prepareParams(query)
fileName = Utils.getNameFromPath(localPath)
files = {'log_file': (fileName, open(tempFile, 'rb'), {'Expires': '0'})}
headers = self.getHeaders()
if SyncConfiguration.EnableProxy:
response = httpSession.post(url, params=query, files=files, timeout=600, headers=headers, proxies=self.getProxies(), verify=False)
elif Utils.isMacOS():
response = httpSession.post(url, params=query, files=files, timeout=600, headers=headers, proxies=self.getProxies(), verify=Utils.getCertFile())
else:
response = httpSession.post(url, params=query, files=files, timeout=600, headers=headers, proxies=self.getProxies())
if response.status_code == StatusCode.Success:
responseJson = response.json()
if responseJson.get('success') == True:
return True
raise WebAPIException
except Exception as exception:
if response is not None:
self.logger.error('[HTTP] Error response content from server: %s', response.content)
self.logger.error(exception, exc_info=1)
raise WebAPIException
finally:
httpSession.close()
def postFileWithToken(self, url, localPath, tempPath, token, appName, query = None):
"""
@return: False if failed, json structure if success
"""
response = None
httpSession = requests.Session()
try:
if Utils.isWindowsOS():
fileName = Utils.getNameFromPath(localPath)
tempFileHanlder = open(tempPath, 'rb')
else:
fileName = Utils.getNameFromPath(localPath)
tempFileHanlder = open(tempPath, 'rb')
files = {'file': (fileName, tempFileHanlder, {'Expires': '0'})}
if query is not None:
query = self.prepareParams(query)
if self.logger is not None:
self.logger.debug('[HTTP] POST Url: ' + url)
multipartMonitor = MultipartEncoderMonitor.from_fields(fields=files, callback=None)
multipartMonitor.filePath = localPath
headersToken = {'Content-Type': multipartMonitor.content_type,
'Auth-Token': token}
if SyncConfiguration.EnableProxy:
response = httpSession.post(url, params=query, data=multipartMonitor, headers=headersToken, timeout=600, verify=False)
elif Utils.isMacOS():
response = httpSession.post(url, params=query, data=multipartMonitor, headers=headersToken, timeout=600, verify=Utils.getCertFile())
else:
response = httpSession.post(url, params=query, data=multipartMonitor, headers=headersToken, timeout=600, proxies=self.getProxies())
self.logger.debug('[HTTP] Response status [%s] and content: %s', str(response.status_code), response.content)
if response.status_code == StatusCode.Success:
responseJson = response.json()
if responseJson.get('success') == True:
return (responseJson, None)
else:
return (False, '\xe8\xaf\xb7\xe9\x87\x8d\xe8\xaf\x95')
else:
errorMsg = '\xe8\xaf\xb7\xe9\x87\x8d\xe8\xaf\x95'
if 400 == response.status_code or 500 == response.status_code:
responseJson = response.json()
if -1 != responseJson.find('failed_to_process_request'):
errorMsg = u'\u4e91\u7aef\u6587\u4ef6\u4e0d\u5b58\u5728'
elif 500 == response.status_code:
errorMsg = u'\u60a8\u7684\u7f51\u7edc\u53ef\u80fd\u5b58\u5728\u95ee\u9898\uff0c\u8bf7\u7a0d\u540e\u518d\u8bd5'
else:
errorMsg = u'\u6743\u9650\u4e0d\u8db3\u65e0\u6cd5\u5b8c\u6210\u8be5\u64cd\u4f5c\uff0c\u8bf7\u68c0\u67e5\u60a8\u7684\u534f\u4f5c\u8005\u7b49\u7ea7'
elif 401 == response.status_code:
errorMsg = u'\u7528\u6237\u5df2\u9000\u51fa' + appName
return (False, errorMsg)
except Exception as exception:
self.logger.error(exception, exc_info=1)
finally:
httpSession.close()
if locals().has_key('tempFileHanlder'):
tempFileHanlder.close()
return (False, '\xe8\xaf\xb7\xe9\x87\x8d\xe8\xaf\x95')
def postFile(self, url, localPath, tempPath, query = None):
"""
@return: False if failed, json structure if success
"""
response = None
httpSession = requests.Session()
bConnectError = False
try:
fileName = Utils.getNameFromPath(localPath)
tempFileHanlder = open(tempPath, 'rb')
files = {'file': (fileName, tempFileHanlder, {'Expires': '0'})}
if query is not None:
query = self.prepareParams(query)
if self.logger is not None:
self.logger.debug('[HTTP] POST Url: ' + url)
multipartMonitor = MultipartEncoderMonitor.from_fields(fields=files, callback=handleChunkFinish)
multipartMonitor.filePath = localPath
headers = self.getHeaders()
headers['Content-Type'] = multipartMonitor.content_type
if SyncConfiguration.EnableProxy:
response = httpSession.post(url, params=query, data=multipartMonitor, headers=headers, timeout=600, proxies=self.getProxies(), verify=False)
elif Utils.isMacOS():
response = httpSession.post(url, params=query, data=multipartMonitor, headers=headers, timeout=600, proxies=self.getProxies(), verify=Utils.getCertFile())
else:
response = httpSession.post(url, params=query, data=multipartMonitor, headers=headers, timeout=600, proxies=self.getProxies())
self.logger.debug('[HTTP] Response status [%s] and content: %s', str(response.status_code), response.content)
if response.status_code == StatusCode.Success:
responseJson = response.json()
if responseJson.get('success') == True:
return responseJson
else:
return False
else:
return self.handleFailedResponseStatus(response.url, response.status_code, response.content)
except (ProxyConnectionError, ConnectionError) as exception:
self.logger.debug('[HTTP] post file stopped because connection abort')
bConnectError = True
self.logger.error(exception, exc_info=1)
raise exception
except (UnAuthorizedException,
RefreshTokenFailedException,
HttpQueryParameterError,
StopSyncing) as exception:
self.logger.debug('[HTTP] stop send file to server')
raise exception
except Exception as exception:
if response is not None:
self.logger.error('[HTTP] Error response content from server: %s', response.content)
self.logger.error(exception, exc_info=1)
raise WebAPIException
finally:
SyncUiInterface.SetNetworkAvailable(not bConnectError)
HttpStatistic.FinishedUploadByte(localPath)
httpSession.close()
if locals().has_key('tempFileHanlder'):
tempFileHanlder.close()
def uploadNewFileV2(self, localPath, tempFilePath, parentFolderId, description = None):
"""
\xe8\xaf\xb4\xe6\x98\x8e: \xe5\x8a\xa0\xe4\xb8\x8a\xe4\xba\x86\xe6\x96\x87\xe4\xbb\xb6\xe5\x88\x86\xe5\x9d\x97\xe4\xb8\x8a\xe4\xbc\xa0\xe5\xa4\x84\xe7\x90\x86,\xe7\xbb\xbc\xe5\x90\x88\xe4\xba\x86\xe6\x96\x87\xe4\xbb\xb6\xe5\x88\x86\xe5\x9d\x97\xe4\xb8\x8a\xe4\xbc\xa0\xe5\x92\x8c\xe6\x96\x87\xe4\xbb\xb6\xe6\x99\xae\xe9\x80\x9a\xe4\xbc\xa0\xe8\xbe\x93\xe5\x8a\x9f\xe8\x83\xbd
@param localPath:
@param tempFilePath:
@param parentFolderId:
@param description:
@return:
"""
try:
self.logger.debug('[Adapter] enter multiupload new file main process, file (%s)', localPath)
if os.path.exists(localPath):
if localPath != tempFilePath:
self.logger.debug('[Adapter] Copy local file (%s) to temp file (%s)', localPath, tempFilePath)
shutil.copyfile(localPath, tempFilePath)
else:
raise WaitingOtherResourceException
sha1 = Utils.sha1ForFile(localPath)
checkExistResponse = self.autoRepeatQuery(self.getFangCloudHttpClient().checkFileExistsOnServerAtTheSameDirWithTheSameName, [localPath,
parentFolderId,
sha1,
description])
if checkExistResponse['success'] == False or not checkExistResponse['is_file_existed']:
localId1, localId2 = self.localFileSystem.getResourceIndex(localPath)
self.__checkFileSha1(localId1, localId2, sha1)
isInterruptUpload, uploadGetResponse = self.__isInterruptUpload(localId1, localId2)
if isInterruptUpload:
if uploadGetResponse.get('success', False):
return self.__multiInterruptUploadNewFile(localPath, tempFilePath, localId1, localId2, uploadGetResponse, sha1)
else:
return uploadGetResponse
else:
isMultiUpLoadNewFile, response = self.__isMultiUploadNewFile(localPath, parentFolderId, sha1)
if response.get('success', False) and isMultiUpLoadNewFile:
return self.__multiUploadNewFile(localPath, tempFilePath, response, sha1)
elif response.get('success', False) and not isMultiUpLoadNewFile:
self.logger.debug('[Adapter] upload file in original way file (%s)', localPath)
return self.__uploadNewFileWithoutPresign(localPath, tempFilePath, response)
else:
return response
else:
self.logger.debug('[Adapter] File (%s) exist on server', localPath)
fileId = -checkExistResponse['file_info']['id']
checkExistResponse['file_info']['success'] = checkExistResponse['success']
checkExistResponse['file_info']['exist'] = True
checkExistResponse['file_info']['is_synced'] = checkExistResponse['is_synced']
checkExistResponse['file_info']['unencrypted_sha1'] = Utils.sha1ForFile(localPath)
ServerResourceAttribute.CreateTime[fileId] = checkExistResponse.get('created_at', time())
ServerResourceAttribute.ModifyTime[fileId] = checkExistResponse.get('modified_at', time())
response = checkExistResponse['file_info']
return self.__formatUploadNewFileResponse(response)
except CheckSumExpiredException:
self.__checkSumExpiredRecovery(localId1, localId2)
raise OperationRetryException
def __computeSha1ForAllBlocks(self, localPath, localId1, localId2, blockNum, blockSize):
"""
\xe8\xae\xa1\xe7\xae\x97\xe4\xb8\x8a\xe4\xbc\xa0\xe6\x96\x87\xe4\xbb\xb6\xe6\xaf\x8f\xe5\x9d\x97\xe7\x9a\x84sha1
@param localPath:
@param blockNum:
@param blockSize:
@return:
"""
if not os.path.exists(localPath):
return
try:
fileSize = os.path.getsize(localPath)
fileToRead = open(localPath, 'rb')
keyForFile = str(localId1) + '_' + str(localId2)
SyncingFileList.SyncingSha1ForFileBlocks[keyForFile] = {}
for i in range(1, blockNum + 1, 1):
start, realBlockSize = self.__getFileStartPosAndBlockSize(fileSize, blockNum, blockSize, i)
sha1ForBlock = Utils.sha1ForBlock(fileToRead, start, realBlockSize)
SyncingFileList.SyncingSha1ForFileBlocks[keyForFile][i] = sha1ForBlock
except IOError as exception:
return
finally:
if locals().has_key('fileToRead'):
fileToRead.close()
def uploadNewFilePresign(self, parentFolderId, localPath, description = None):
try:
postBody = {'folder_id': parentFolderId,
'file_name': Utils.getNameFromPath(localPath),
'file_size': str(os.path.getsize(localPath)),
'modified_at': Utils.getModifyTime(localPath)}
if description is not None:
postBody['description'] = description
response = self.post(FangCloudUrl.URL['FILE_PRESIGN_UPLOAD'], jsonData=postBody, logData=True)
self.callbackHandler.handleHttpResponse(response)
return response
except UnAuthorizedException as exception:
self.logger.error(exception, exc_info=1)
return self.__refreshTokenWhenUnAuthorized()
except ConnectionError:
return HttpResponseResult.Failed
def startMultiUploadNewFileVersion(self, localPath, fileId, sequenceId, sha1):
"""
\xe8\xaf\xb4\xe6\x98\x8e: \xe5\x8f\x91\xe9\x80\x81\xe5\xbc\x80\xe5\xa7\x8b\xe5\x88\x86\xe5\x9d\x97\xe4\xb8\x8a\xe4\xbc\xa0\xe8\xaf\xb7\xe6\xb1\x82-\xe9\x92\x88\xe5\xaf\xb9\xe6\x96\x87\xe4\xbb\xb6\xe6\x96\xb0\xe7\x89\x88\xe6\x9c\xac
@param localPath:
@param fileId:
@param sequenceId:
@param sha1:
@return:
"""
try:
startPostBody = {'sha1': sha1,
'file_size': str(os.path.getsize(localPath)),
'file_name': os.path.basename(localPath),
'sequence_id': sequenceId,
'remark': '',
'folder_id': '',
'modified_at': Utils.getModifyTime(localPath)}
print startPostBody
url = FangCloudUrl.URL['FILE_MULTI_UPLOAD_WITH_FILE_ID'] % fileId
response = self.post(url, jsonData=startPostBody, logData=True)
self.callbackHandler.handleHttpResponse(response)
return response
except UnAuthorizedException as exception:
self.logger.error(exception, exc_info=1)
return self.__refreshTokenWhenUnAuthorized()
except ConnectionError:
return HttpResponseResult.Failed
def getSynedFolderChildren(self, folderId, isSyncedFolder = True):
isSecurityTokenExpired = True
while isSecurityTokenExpired:
isSecurityTokenExpired = False
responseIds = self.autoRepeatQuery(self.getFangCloudHttpClient().getSynedFolderChildrenIds, [folderId, isSyncedFolder])
filesRtn = []
foldersRtn = []
if responseIds.get('success', False) == True:
pageSize = 5000
foldersSplit = Utils.splitArrayByFixedSize(responseIds['folders'], pageSize)
filesSplit = Utils.splitArrayByFixedSize(responseIds['files'], pageSize)
totalLen = 0
for curIndex1 in filesSplit:
totalLen += len(curIndex1)
isSuccess = True
if len(filesSplit) > len(foldersSplit):
lenMax = len(filesSplit)
else:
lenMax = len(foldersSplit)
for iCur in range(0, lenMax):
foldersRequest = []
filesRequest = []
if iCur < len(filesSplit):
filesRequest = filesSplit[iCur]
if iCur < len(foldersSplit):
foldersRequest = foldersSplit[iCur]
bulkTemp = self.autoRepeatQuery(self.getFangCloudHttpClient().getSyncedBulkInfo, [foldersRequest, filesRequest, responseIds['secure_token']])
if bulkTemp.get('success', False) == False:
isSuccess = False
if bulkTemp.get('error', '') == 'sync_bulk_get_invalid':
isSecurityTokenExpired = True
break
filesRtn = filesRtn + bulkTemp['files']
foldersRtn = foldersRtn + bulkTemp['folders']
else:
isSuccess = False
if isSecurityTokenExpired:
sleep(1)
continue
if isSuccess:
return (foldersRtn, filesRtn)
return ([], [])
def refreshAuthToken(self, forceRereshToken = False):
"""
Response:
{
"auth_token": "a8f5f167f44f4964e6c998dee827110c",
"refresh_token": "964e6c998dee827110ca8f5f167f44f4",
"expires_at": "1401698962",
}
@return: False if failed, json structure if success
"""
try:
TOKEN_REFRESH_LOCK.acquire()
if HttpBasicClient.UserRefreshToken is None:
self.logger.warning('[HTTP] refresh token stored in cache is invalidate')
return HttpResponseResult.RefreshTokenIgnore
if type(HttpBasicClient.ExpireDate) == int:
self.logger.warning('[HTTP] error of expire date: %d', HttpBasicClient.ExpireDate)
currentTime = self.getServerCurrentTime()
if HttpBasicClient.ExpireDate is not None and currentTime < HttpBasicClient.ExpireDate - datetime.timedelta(seconds=60) and not forceRereshToken:
self.logger.debug('[HTTP] do not need to refresh token because other thread have finished it, the expire date is %s', HttpBasicClient.ExpireDate)
return HttpResponseResult.RefreshTokenIgnore
postBody = {'refresh_token': HttpBasicClient.UserRefreshToken}
response = self.post(FangCloudUrl.URL['RERRESH_AUTH_TOKEN'], jsonData=postBody, logData=True)
if response != False:
HttpBasicClient.UserAuthToken = response['auth_token']
HttpBasicClient.UserRefreshToken = response['refresh_token']
HttpBasicClient.ExpireDate = datetime.datetime.fromtimestamp(int(response['expires_at']))
Utils.storeAuthToken(HttpBasicClient.UserAuthToken)
UserDatabaseManager().refreshAuthToken(response['auth_token'], response['refresh_token'], response['expires_at'])
else:
self.logger.info('[HTTP] Refresh user token failed')
self.callbackHandler.handleHttpResponse(response)
return response
except UnAuthorizedException as exception:
self.logger.error(exception, exc_info=1)
return HttpResponseResult.Failed
except ConnectionError:
return HttpResponseResult.Failed
finally:
TOKEN_REFRESH_LOCK.release()
def login(self, username, password):
"""
@return: return false if exception or failed. return the following user structure if success.
{
"auth_token": "a8f5f167f44f4964e6c998dee827110c",
"refresh_token": "964e6c998dee827110ca8f5f167f44f4",
"auth_success": true,
"expires_at": "1401698962",
"user": {
"type": "user",
"id": "12312311",
"name": "Yuan Cheng",
"login": "******",
"phone": "123123",
"space_total": "123123",
"space_used": "1123123",
"name_first_letter": "a",
"profile_pic_key" : "ksui667f44f4964e6c998dee8245s61",
},
}
"""
try:
urlQuery = {'api_key': API_KEY}
postBody = {'login': username,
'password': password}
response = self.post(FangCloudUrl.URL['USER_LOGIN'], query=urlQuery, jsonData=postBody, logData=False)
if response != False:
HttpBasicClient.UserAuthToken = response['auth_token']
HttpBasicClient.UserRefreshToken = response['refresh_token']
HttpBasicClient.ExpireDate = datetime.datetime.fromtimestamp(int(response['expires_at']))
Utils.storeAuthToken(HttpBasicClient.UserAuthToken)
else:
self.logger.info('[HTTP] User login failed!')
self.callbackHandler.handleHttpResponse(response)
return response
except UnAuthorizedException as exception:
self.logger.error(exception, exc_info=1)
return HttpResponseResult.Failed
except ConnectionError:
return HttpResponseResult.Failed
def getHeaders(self, additional = None):
if HttpBasicClient.UserAuthToken is not None:
headers = {'Egeio-Client-Info': SyncConfiguration.ClientInfo,
'Auth-Token': HttpBasicClient.UserAuthToken,
'Device-Token': SyncConfiguration.DeviceToken,
'X-Custom-Productid': '%s' % Utils.getSyncProductId()}
else:
headers = {'Egeio-Client-Info': SyncConfiguration.ClientInfo}
if additional is not None:
headers.update(additional)
self.logger.debug('[HttpBasicClient] http header = %s', headers)
return headers
def postStreamFile(self, url, filePath, tempPath, start, limit, query = None, header = {}, resetUploadedByte = False):
response = None
httpSession = requests.Session()
bConnectError = False
try:
if query is not None:
query = self.prepareParams(query)
if self.logger is not None:
self.logger.debug('[HTTP] POST Url: ' + url)
header = self.getHeaders(header)
it = UploadInChunks(filePath, tempPath, 10240, start, limit)
if SyncConfiguration.EnableProxy:
response = httpSession.post(url, params=query, data=IterableToFileAdapter(it), headers=header, timeout=600, proxies=self.getProxies(), verify=False)
elif Utils.isMacOS():
response = httpSession.post(url, params=query, data=IterableToFileAdapter(it), headers=header, timeout=600, proxies=self.getProxies(), verify=Utils.getCertFile())
else:
response = httpSession.post(url, params=query, data=IterableToFileAdapter(it), headers=header, timeout=600, proxies=self.getProxies())
self.logger.debug('[HTTP] Response status [%s] and content: %s', str(response.status_code), response.content)
if response.status_code == StatusCode.Success:
responseJson = response.json()
if responseJson.get('success'):
return responseJson
else:
return False
else:
return self.handleFailedResponseStatus(response.url, response.status_code, response.content)
except (ProxyConnectionError, ConnectionError) as exception:
self.logger.debug('[HTTP] post file stopped because connection abort')
self.logger.error(exception, exc_info=1)
bConnectError = True
raise exception
except (StopSyncing, StopSyncingAndRestart) as exception:
self.logger.debug('[HTTP] stop send file to server and clear upload byte')
HttpStatistic.FinishedUploadByte(filePath)
raise exception
except (UnAuthorizedException,
RefreshTokenFailedException,
HttpQueryParameterError,
CheckSumExpiredException) as exception:
self.logger.debug('[HTTP] stop send file to server')
raise exception
except Exception as exception:
if response is not None:
self.logger.error('[HTTP] Error response content from server: %s', response.content)
self.logger.error(exception, exc_info=1)
raise WebAPIException
finally:
SyncUiInterface.SetNetworkAvailable(not bConnectError)
if resetUploadedByte:
HttpStatistic.FinishedUploadByte(filePath)
httpSession.close()
def get(self, url, query = None, logData = True):
"""
@return: False if failed, json structure if success
"""
response = None
httpSession = requests.Session()
bConnectError = False
try:
if query is not None:
query = self.prepareParams(query)
self.logger.debug('[HTTP] GET Url: ' + url)
while True:
try:
if SyncConfiguration.EnableProxy:
response = httpSession.get(url, params=query, headers=self.getHeaders(), timeout=600, proxies=self.getProxies(), verify=False)
elif Utils.isMacOS():
response = httpSession.get(url, params=query, headers=self.getHeaders(), timeout=600, proxies=self.getProxies(), verify=Utils.getCertFile())
else:
response = httpSession.get(url, params=query, headers=self.getHeaders(), timeout=600, proxies=self.getProxies())
break
except requests.exceptions.Timeout:
self.logger.warning('[HTTP] time out and try the request again')
if logData:
self.logger.debug('[HTTP] Response status [%s] and content: %s', str(response.status_code), response.content)
else:
self.logger.debug('[HTTP] Response status [%s]', str(response.status_code))
if response.status_code == StatusCode.Success:
responseJson = json.loads(response.content)
if responseJson.get('success') == True:
return responseJson
else:
return False
else:
return self.handleFailedResponseStatus(response.url, response.status_code, response.content)
except (ProxyConnectionError, ConnectionError) as exception:
bConnectError = True
self.logger.error(exception, exc_info=1)
raise exception
except (UnAuthorizedException, RefreshTokenFailedException, HttpQueryParameterError) as exception:
self.logger.error(exception, exc_info=1)
raise exception
except Exception as exception:
if response is not None:
self.logger.error('[HTTP] Error response content from server: %s', response.content)
self.logger.error(exception, exc_info=1)
raise WebAPIException
finally:
SyncUiInterface.SetNetworkAvailable(not bConnectError)
httpSession.close()
def getFileWithToken(self, url, localPath, token, query = None):
"""
@return: False if failed, json structure if success
"""
response = None
httpSession = requests.Session()
try:
if query is not None:
query = self.prepareParams(query)
self.logger.debug('[HTTP] GET Url: %s; file path: %s', url, localPath)
if None != token:
headersToken = {'Auth-Token': token}
else:
headersToken = {}
if SyncConfiguration.EnableProxy:
response = httpSession.get(url, params=query, headers=headersToken, timeout=600, stream=True, verify=False)
elif Utils.isMacOS():
response = httpSession.get(url, params=query, headers=headersToken, timeout=600, stream=True, verify=Utils.getCertFile())
else:
response = httpSession.get(url, params=query, headers=headersToken, timeout=600, stream=True)
self.logger.debug('[HTTP] Response status [%s]', str(response.status_code))
if response.status_code == StatusCode.Success:
with open(localPath, 'wb') as f:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
return (True, None, HttpTokenExitCode.DOWNLOAD_OK)
errorCode = HttpTokenExitCode.DOWNLOAD_RETRY_ERROR
strError = '\xe8\xaf\xb7\xe9\x87\x8d\xe8\xaf\x95'
if 400 == StatusCode.Success:
errorCode = HttpTokenExitCode.DOWNLOAD_ACCESS_ERROR
strError = u'\u6743\u9650\u4e0d\u8db3\u65e0\u6cd5\u5b8c\u6210\u8be5\u64cd\u4f5c\uff0c\u8bf7\u68c0\u67e5\u60a8\u7684\u534f\u4f5c\u8005\u7b49\u7ea7'
elif 500 == StatusCode.Success:
errorCode = HttpTokenExitCode.DOWNLOAD_NET_ERROR
strError = u'\u60a8\u7684\u7f51\u7edc\u53ef\u80fd\u5b58\u5728\u95ee\u9898\uff0c\u8bf7\u7a0d\u540e\u518d\u8bd5'
elif 401 == StatusCode.Success:
errorCode = HttpTokenExitCode.DOWNLOAD_TOKEN_EXPIRED
strError = u'\u7528\u6237\u5df2\u6ce8\u9500'
return (False, strError, errorCode)
except Exception as exception:
if response is not None:
self.logger.error('[HTTP] Error response content from server: %s', response.content)
self.logger.error(exception, exc_info=1)
finally:
httpSession.close()
return (False, '\xe8\xaf\xb7\xe9\x87\x8d\xe8\xaf\x95', HttpTokenExitCode.DOWNLOAD_RETRY_ERROR)
def checkFileExistsOnServerAtTheSameDirWithTheSameName(self, localPath, parentFolderId, sha1, description = None):
try:
_, filename = os.path.split(localPath)
urlQuery = {'folder_id': parentFolderId,
'file_name': filename,
'unencrypted_sha1': sha1,
'modified_at': Utils.getModifyTime(localPath)}
if description is not None:
urlQuery['description'] = description
response = self.post(FangCloudUrl.URL['FILE_UPLOAD_CHECK_FILE_EXISTS_IN_SAME_DIR_IN_SAME_NAME'], None, urlQuery)
self.callbackHandler.handleHttpResponse(response)
return response
except UnAuthorizedException as exception:
self.logger.error(exception, exc_info=1)
return self.__refreshTokenWhenUnAuthorized()
except ConnectionError:
return HttpResponseResult.Failed
def getFileForUpdate(self, url, localPath, query = None):
"""
@return: False if failed, json structure if success
"""
response = None
httpSession = requests.Session()
try:
if query is not None:
query = self.prepareParams(query)
self.logger.debug('[HTTP] GET Url: %s; file path: %s', url, localPath)
if SyncConfiguration.EnableProxy:
response = httpSession.get(url, params=query, headers=self.getHeaders(), stream=True, proxies=self.getProxies(), verify=False)
elif Utils.isMacOS():
response = httpSession.get(url, params=query, headers=self.getHeaders(), stream=True, proxies=self.getProxies(), verify=Utils.getCertFile())
else:
response = httpSession.get(url, params=query, headers=self.getHeaders(), stream=True, proxies=self.getProxies())
self.logger.debug('[HTTP] Response status [%s]', str(response.status_code))
if response.status_code == StatusCode.Success:
with open(localPath, 'wb') as f:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
return True
return self.handleFailedResponseStatus(response.url, response.status_code, response.content)
except IOError as exception:
self.logger.error(exception, exc_info=1)
raise exception
except (UnAuthorizedException,
RefreshTokenFailedException,
HttpQueryParameterError,
StopSyncing,
ConnectionError) as exception:
raise exception
except Exception as exception:
if response is not None:
self.logger.error('[HTTP] Error response content from server: %s', response.content)
self.logger.error(exception, exc_info=1)
raise WebAPIException
finally:
httpSession.close()
def uploadNewFileVersionV2(self, localPath, tempFilePath, resourceId, sequenceId, description = 'UpdateNewVersion'):
"""
\xe8\xaf\xb4\xe6\x98\x8e: \xe4\xb8\x8a\xe4\xbc\xa0\xe6\x96\x87\xe4\xbb\xb6\xe6\x96\xb0\xe7\x89\x88\xe6\x9c\xac\xe6\x8e\xa5\xe5\x8f\xa3,\xe7\xbb\xbc\xe5\x90\x88\xe4\xba\x86\xe6\x96\x87\xe4\xbb\xb6\xe5\x88\x86\xe5\x9d\x97\xe4\xb8\x8a\xe4\xbc\xa0\xe5\x92\x8c\xe6\x96\x87\xe4\xbb\xb6\xe6\x99\xae\xe9\x80\x9a\xe4\xb8\x8a\xe4\xbc\xa0
@param localPath:
@param tempFilePath:
@param resourceId:
@param sequenceId:
@param description:
@return:
"""
try:
self.logger.debug('[Adapter] enter multiupload new version main process, file (%s)', localPath)
self.treeDatabase.getResourceByServerId(resourceId)
if os.path.exists(localPath):
if localPath != tempFilePath:
self.logger.debug('[Adapter] Copy local file (%s) to temp file (%s)', localPath, tempFilePath)
shutil.copyfile(localPath, tempFilePath)
else:
raise WaitingOtherResourceException
sha1 = Utils.sha1ForFile(localPath)
treeResource = self.treeDatabase.getResourceByServerId(resourceId)
localId1 = treeResource.local_id1
localId2 = treeResource.local_id2
self.__checkFileSha1(localId1, localId2, sha1)
isInterruptUploadOrNot, uploadGetResponse = self.__isInterruptUpload(localId1, localId2)
if isInterruptUploadOrNot:
if uploadGetResponse.get('success', False):
return self.__multiInterruptUploadNewFile(localPath, tempFilePath, localId1, localId2, uploadGetResponse, sha1)
else:
return uploadGetResponse
else:
isMultiUpLoadNewFileOrNot, response = self.__isMultiUploadNewVersionOrNot(localPath, -resourceId, sequenceId, sha1)
if response.get('success', False) and isMultiUpLoadNewFileOrNot:
return self.__multiUploadNewFileVersion(localPath, tempFilePath, resourceId, sequenceId, response, sha1)
if response.get('success', False) and not isMultiUpLoadNewFileOrNot:
self.logger.debug('[Adapter] upload new version in original way file (%s)', localPath)
return self.__uploadNewFileVersionWithoutPresign(localPath, tempFilePath, response, description)
return response
except CheckSumExpiredException:
self.__checkSumExpiredRecovery(localId1, localId2)
raise OperationRetryException
def getFile(self, url, localPath, desPath, query = None):
"""
@localPath: the file that need to be upload, this maybe temp file
@desPath: destination position
@return: False if failed, json structure if success
"""
response = None
httpSession = requests.Session()
bConnectError = False
try:
if query is not None:
query = self.prepareParams(query)
self.logger.debug('[HTTP] GET Url: %s; file path: %s', url, localPath)
headers = self.getHeaders()
if os.path.exists(localPath):
bytePos = os.path.getsize(localPath)
headers['RANGE'] = 'bytes=%d-' % bytePos
else:
bytePos = 0
if SyncConfiguration.EnableProxy:
response = httpSession.get(url, params=query, headers=headers, timeout=600, stream=True, proxies=self.getProxies(), verify=False)
elif Utils.isMacOS():
response = httpSession.get(url, params=query, headers=headers, timeout=600, stream=True, proxies=self.getProxies(), verify=Utils.getCertFile())
else:
response = httpSession.get(url, params=query, headers=headers, timeout=600, stream=True, proxies=self.getProxies())
self.logger.debug('[HTTP] Response status [%s]', str(response.status_code))
if response.status_code == StatusCode.Success or response.status_code == StatusCode.PartialContent:
try:
if bytePos > 0:
openTag = 'ab'
else:
openTag = 'wb'
openfile = open(localPath, openTag)
for chunk in response.iter_content(chunk_size=10240):
if chunk:
openfile.write(chunk)
HttpStatistic.AddDownloadByte(localPath, 10240)
if not SyncConfiguration.IsEnableSyncing or SyncConfiguration.IsFullSyncIniting:
raise StopSyncing
if desPath not in SyncingFileList.SyncingFileSet:
raise StopSyncingAndRestart
finally:
if locals().has_key('openfile'):
openfile.close()
if response.status_code == StatusCode.PartialContent and long(response.headers.get('Content-Range').split('/')[1]) != os.path.getsize(localPath):
self.logger.debug('[HTTP] More data should be transfer, status code = %d', response.status_code)
result = False
elif response.status_code == StatusCode.Success:
if response.headers.get('Transfer-Encoding') != 'chunked' and long(response.headers.get('Content-Length')) != os.path.getsize(localPath):
self.logger.debug('[HTTP] More data should be transfer, status code = %d', response.status_code)
result = False
elif response.headers.get('Transfer-Encoding') == 'chunked' and response.headers.get('X-Decrypt-Content-Length') is not None and long(response.headers.get('X-Decrypt-Content-Length')) != os.path.getsize(localPath):
self.logger.debug('[HTTP] More data should be transfer, status code = %d', response.status_code)
result = False
else:
result = True
else:
self.logger.debug('[HTTP] File download successfully')
result = True
return result
if response.status_code == StatusCode.NotFound:
result = {'success': False,
'error': HttpErrorMessage.UnavailableFileOnServer}
return result
return self.handleFailedResponseStatus(response.url, response.status_code, response.content)
except (ProxyConnectionError, ConnectionError) as exception:
self.logger.error(exception, exc_info=1)
bConnectError = True
raise exception
except ReadTimeout as exception:
self.logger.error(exception, exc_info=1)
raise OperationRetryInOneMinuteException
except IOError as exception:
self.logger.error(exception, exc_info=1)
raise exception
except (UnAuthorizedException,
RefreshTokenFailedException,
HttpQueryParameterError,
StopSyncing) as exception:
self.logger.error(exception, exc_info=1)
raise exception
except Exception as exception:
self.logger.error(exception, exc_info=1)
raise WebAPIException
finally:
SyncUiInterface.SetNetworkAvailable(not bConnectError)
httpSession.close()