def generateAuthorizationHeader(url=None, data=None):
signature = ""
if data:
encodedata = urlencode(data)
xxxx = SECRET_KEY + encodedata
md = hashlib.md5(xxxx.encode())
signature = md.hexdigest()
elif url:
urlencode(url)
xx = SECRET_KEY + url
signature = hashlib.md5(xx.encode()).hexdigest()
return {SIGNATURE_HEADER: signature}
def editFolder(self, folder_id, name):
url = self.serverURL + "/clapi/file/editFolder/"
data = {"folder_id": folder_id, "name": name}
self.browser.add_header(extraHeader=generateAuthorizationHeader(data=data))
r = self.browser.open(url, urlencode(data))
html = r.content
return html
def createUploadSession(self, file_hash, file_size, file_name, erfk, folder_id=None):
url = self.serverURL + "/clapi/session/createUploadSession/"
data = {
"file_hash": file_hash,
"file_size": file_size,
"file_name": file_name,
"erfk": reverseString(base64.b64encode(erfk)),
# todo: remember to change this to send reverse base64 to server
}
if folder_id:
data["folder_id"] = folder_id
self.browser.add_header(extraHeader=generateAuthorizationHeader(data=data))
for ti in range(3):
try:
r = self.browser.open(url, urlencode(data))
html = r.content
except Exception as e:
print("retry upload ti=%s due to error=%s" % (ti + 1, e))
time.sleep(0.5)
else: # OK
break
result = json.loads(html.decode("utf-8"))
# print result
upload_session_id = result["session_id"]
print("Upload sesison id=%s" % (upload_session_id))
return result
def deleteFolder(self, *folderIDList):
url = self.serverURL + "/clapi/file/deleteFolder/"
data = [("folder_id", folder_id) for folder_id in folderIDList]
self.browser.add_header(extraHeader=generateAuthorizationHeader(data=data))
r = self.browser.open(url, urlencode(data))
html = r.content
return
def newFolder(self, folder_name, folder_id):
url = self.serverURL + "/clapi/file/newFolder/"
data = {"folder_name": folder_name}
if folder_id: # folder_id==None mean root folder
data["folder_id"] = folder_id
self.browser.add_header(extraHeader=generateAuthorizationHeader(data=data))
r = self.browser.open(url, urlencode(data))
html = r.content
return json.loads(html.decode("utf-8"))["folder_id"]
def moveFile(self, *fileIDList, **kwargs):
url = self.serverURL + "/clapi/file/moveFile/"
folder_id = kwargs["folder_id"]
if folder_id is None:
folder_id = 0
data = [("file_id", file_id) for file_id in fileIDList] + [("folder_id", folder_id)]
self.browser.add_header(extraHeader=generateAuthorizationHeader(data=data))
r = self.browser.open(url, urlencode(data))
html = r.content
def deleteFile(self, *fileIDList):
url = self.serverURL + "/clapi/file/deleteFile/"
data = [("file_id", file_id) for file_id in fileIDList]
self.browser.add_header(extraHeader=generateAuthorizationHeader(data=data))
r = self.browser.open(url, urlencode(data))
html = r.content
if "success" in html.decode("utf-8"):
return True
else:
return
def upload_chunk(
self,
upload_url,
key,
len_data,
getData,
postData,
resumableChunkNumber,
resumableChunkSize,
resumableFilename,
files,
):
uploaded_chunk = True
i = 0
while i < 5:
try:
response = requests.get(upload_url + "?" + urlencode(getData), timeout=30)
html = response.content
except Exception as e:
print(e)
time.sleep(1)
else:
if response.status_code != 200:
print(response.content())
time.sleep(1)
i += 1
continue
break
i += 1
if i >= 5:
uploaded_chunk = False
if (
html.find("chunk uploaded") == -1
and html.find("chunks already exists") == -1
and html.find("file_id=") == -1
):
print("upload chunk")
i = 0
while i < 5:
try:
response = requests.post(upload_url, data=postData, files=files)
# response = requests.post(upload_url, data=mem, headers={'Content-Type': mem.content_type})
html = response.content
except Exception as e:
print(e)
time.sleep(1)
else:
# print response.status_code
if response.status_code != 200:
print("upload_3 failed with response=%s" % (response.content))
# c = open('error3.html','w')
# c.write(response.text)
# c.close()
time.sleep(1)
i += 1
continue
else:
break
i += 1
if i >= 5:
uploaded_chunk = False
print("upload response=%s" % (html))
if uploaded_chunk:
self.processQueue.put({"uploaded": len_data})
else:
self.processQueue.put(
{
"upload_error": (
upload_url,
key,
len_data,
getData,
postData,
resumableChunkNumber,
resumableChunkSize,
resumableFilename,
files,
)
}
)
self.processQueue.put("upload ready")
return
def deleteUserFile(self, userFile_id):
url = self.serverURL + "/clapi/file/deleteFile/"
data = {"file_id": userFile_id}
self.browser.add_header(extraHeader=generateAuthorizationHeader(data=data))
r = self.browser.open(url, urlencode(data))
html = r.content
def encryptResumableUpload(
self,
upload_url,
filePath,
resumableIdentifier,
file_size,
key="000102030405060708090a0b0c0d0e0f",
progressFunction=None,
):
self.uploadQueque = Queue()
self.processQueue = Queue()
self.thread_upload_chunk = []
resumableTotalSize = os.path.getsize(filePath)
# resumableChunkSize = 512 * 1024 # 1MB
if resumableTotalSize < 50 * 1024 * 1024:
resumableChunkSize = 1 * 1024 * 1024 # 1MB
else:
resumableChunkSize = 10 * 1024 * 1024 # 10MB
resumableTotalChunks = file_size / resumableChunkSize
if resumableTotalChunks == 0:
resumableTotalChunks = 1
resumableFilename = os.path.split(filePath)[1].encode("utf-8")
resumableRelativePath = filePath.encode("utf-8")
key = "".join([chr(int(key[i : i + 2], 16)) for i in range(0, len(key), 2)])
# print 'filekey:'+str(key)
f = open(filePath, "rb")
result = None
resumableChunkNumber = 1
self.uploadedSize = 0
# last_data = 0
start_time = time.time()
last_data = False
self.stop = False
while self.uploadedSize < resumableTotalSize:
if self.stop:
result = None
break
if (
resumableChunkNumber < resumableTotalChunks
): # if uploadedSize <= resumableTotalSize - 2 * resumableChunkSize:
data = f.read(resumableChunkSize)
else:
print("last data")
print(resumableIdentifier)
print(resumableTotalChunks)
print(resumableChunkNumber)
print(resumableFilename)
data = f.read()
print(len(data))
last_data = True
# print last_data
print("len:" + str(len(data)))
if len(data) == 0:
result = None
break
postData = {
"resumableChunkNumber": str(resumableChunkNumber),
"resumableChunkSize": str(resumableChunkSize),
"resumableTotalSize": str(resumableTotalSize),
"resumableTotalChunks": str(resumableTotalChunks),
"resumableFilename": str(resumableFilename),
"resumableIdentifier": str(resumableIdentifier),
"resumableRelativePath": str(resumableRelativePath),
"resumableCurrentChunkSize": str(len(data)),
"resumableType": "",
}
getData = {
"resumableChunkNumber": resumableChunkNumber,
"resumableChunkSize": resumableChunkSize,
"resumableTotalSize": resumableTotalSize,
"resumableCurrentChunkSize": str(len(data)),
"resumableType": "",
"resumableTotalChunks": resumableTotalChunks,
"resumableIdentifier": resumableIdentifier,
"resumableFilename": resumableFilename,
"resumableRelativePath": resumableRelativePath,
}
if not last_data:
# print 'upload Chunk'
if resumableTotalChunks < 5:
totalopenThead = resumableTotalChunks
else:
totalopenThead = 5
if totalopenThead == 1:
totalopenThead = 0
while len(self.thread_upload_chunk) < totalopenThead:
print("Creat Upload Chunk Thread")
th = Upload_chunk(self.uploadQueque, self.processQueue)
th.start()
self.thread_upload_chunk.append(th)
while len(self.thread_upload_chunk):
try:
uploaded_data = self.processQueue.get_nowait()
except:
pass
else:
if uploaded_data == "upload ready":
counter_value = (resumableChunkNumber - 1) * resumableChunkSize / 16
ctr = Counter.new(128, initial_value=1 + counter_value)
# print ctr
cipher = AES.new(key, AES.MODE_CTR, counter=ctr)
# encode_data = cipher.encrypt(data)
# print len(data)
files = {
"file": (
resumableFilename.decode("utf-8").encode("ascii", "ignore"),
cipher.encrypt(data),
)
}
# postData.update({
# 'file': (resumableFilename.decode("utf-8").encode('ascii', 'ignore'), cipher.encrypt(data))
# })
self.uploadQueque.put(
(
upload_url,
key,
len(data),
getData,
postData,
resumableChunkNumber,
resumableChunkSize,
resumableFilename,
files,
)
)
# del postData
# del encode_data
# del data
resumableChunkNumber += 1
break
elif uploaded_data.has_key("callback"):
self.uploadedSize += uploaded_data["callback"]
uploaded_time = time.time() - start_time
upload_speed = self.uploadedSize / uploaded_time
progressFunction(0, 0, resumableChunkSize, self.uploadedSize, speed=int(upload_speed))
elif uploaded_data.has_key("uploaded"):
self.uploadedSize += uploaded_data["uploaded"]
uploaded_time = time.time() - start_time
upload_speed = self.uploadedSize / uploaded_time
progressFunction(0, 0, resumableChunkSize, self.uploadedSize, speed=int(upload_speed))
elif uploaded_data.has_key("upload_error"):
data_error = uploaded_data["upload_error"]
upload_url, key, len_data, getData, postData, resumableChunkNumber, resumableChunkSize, resumableFilename = (
data_error
)
# reset uploaded data
self.uploadedSize -= len_data
#
self.uploadQueque.put(
(
upload_url,
key,
len_data,
getData,
postData,
resumableChunkNumber,
resumableChunkSize,
resumableFilename,
)
)
else:
result = None
break
else:
stopped_th = 0
while len(self.thread_upload_chunk):
if len(self.thread_upload_chunk) == stopped_th:
break
try:
uploaded_data = self.processQueue.get_nowait()
except:
pass
else:
if uploaded_data == "upload ready":
self.uploadQueque.put("stop")
elif uploaded_data == "stopped":
stopped_th += 1
if len(self.thread_upload_chunk) == stopped_th:
break
elif uploaded_data.has_key("callback"):
self.uploadedSize += uploaded_data["callback"]
uploaded_time = time.time() - start_time
upload_speed = self.uploadedSize / uploaded_time
progressFunction(0, 0, resumableChunkSize, self.uploadedSize, speed=int(upload_speed))
elif uploaded_data.has_key("uploaded"):
self.uploadedSize += uploaded_data["uploaded"]
uploaded_time = time.time() - start_time
upload_speed = self.uploadedSize / uploaded_time
progressFunction(0, 0, resumableChunkSize, self.uploadedSize, speed=int(upload_speed))
elif uploaded_data.has_key("upload_error"):
data_error = uploaded_data["upload_error"]
upload_url, key, len_data, getData, postData, resumableChunkNumber, resumableChunkSize, resumableFilename = (
data_error
)
# reset uploaded data
self.uploadedSize -= len_data
self.uploadQueque.put(
(
upload_url,
key,
len_data,
getData,
postData,
resumableChunkNumber,
resumableChunkSize,
resumableFilename,
)
)
if self.stop:
break
print("Check file")
i = 0
while i < 5:
try:
response = requests.get(upload_url + "?" + urlencode(getData), timeout=30)
html = response.content
except Exception as e:
print(e)
time.sleep(1)
else:
if response.status_code != 200:
print(response.content)
time.sleep(1)
i += 1
continue
break
i += 1
if i >= 5:
result = None
break
# print response.content
if (
html.find("chunk uploaded") == -1
and html.find("chunks already exists") == -1
and html.find("file_id=") == -1
):
print("uploading chunk")
i = 0
while i < 5:
try:
response = requests.post(upload_url, data=postData, files=files)
html = response.content
except Exception as e:
print(e)
time.sleep(1)
else:
if response.status_code != 200:
print("upload_3 failed with response=%s" % response.content)
# c = open('error3.html','w')
# c.write(response.text)
# c.close()
time.sleep(1)
i += 1
continue
else:
break
i += 1
if i >= 5:
result = None
break
print("upload done")
resumableChunkNumber += 1
# else:
self.uploadedSize += len(data)
print("upload response=%s" % (html))
# print '===='
# print self.uploadedSize
# print resumableTotalSize
if self.uploadedSize >= resumableTotalSize or resumableChunkNumber >= resumableTotalChunks + 1:
print(html)
if self.uploadedSize > resumableTotalSize:
print("error")
print(self.uploadedSize, resumableTotalSize)
result = response.text.split("=")[1]
break
# time.sleep(0.5)
f.close()
return result
