def splitFile(inputFile):
inputFilePath = os.path.abspath(inputFile)
kilobytes = 1024
Megabytes = kilobytes * 1024
chunkSize = int(Megabytes * 8)
indexOfPoint = inputFilePath.rfind('.')
indexOfSlash = inputFilePath.rfind('/')
fileName = inputFilePath[indexOfSlash+1:indexOfPoint]
# read input file
if os.path.isfile(inputFilePath):
f = open(inputFilePath, 'rb')
data = f.read()
f.close()
print ("\n[SYSTEM] Finished read binary file for upload")
# get the length of data, ie size of the input file in bytes
bytes = len(data)
# Calculate the number of chunks to be created
if bytes < chunkSize:
chunkSize = int(kilobytes * 512)
noOfChunks = bytes / chunkSize
if (bytes % chunkSize):
noOfChunks += 1
else:
noOfChunks = bytes / chunkSize
if (bytes % chunkSize):
noOfChunks += 1
else:
while (noOfChunks > 41):
chunkSize *= 2
noOfChunks = bytes / chunkSize
if (bytes % chunkSize):
noOfChunks += 1
# Initialize JSON text variable
indexOfChunk = 0
chunkInfoList = []
used_origin_credential = []
used_replication_credential = []
circle = 0
idx = 0
addGroupIdx = 0
# Received All Credential List
# Received All Credential Dictionary divided by group
receivedCredential, credentials_list = googleDrive.get_credentials_list()
uploadfilePath = os.getcwd() + "/cache"
if not os.path.isdir(uploadfilePath):
os.makedirs(uploadfilePath)
uploadfilePath = uploadfilePath + '/'
for i in range(0, bytes + 1, chunkSize):
#fn1 = fileName + "%s" % i
idx += 1
fn1 = fileName + '_' + "%s" % idx
f = open(uploadfilePath+fn1, 'wb')
f.write(data[i: i + chunkSize])
f.close()
print ("[CREATE] Created chunk file({0}) - '{1}'".format(indexOfChunk+1, fn1))
# Set to upload google drive account
if indexOfChunk < len(credentials_list):
credentialIndex = indexOfChunk
else:
credentialIndex = indexOfChunk % len(credentials_list)
circle += 1
# Input metadata of chunk file in dictionary
dict = {}
dict['fileName'] = inputFilePath[indexOfSlash+1:]
dict['numberOfChunks'] = int(noOfChunks)
dict['indexOfChunk'] = indexOfChunk
dict['chunkName'] = fn1
file_dict = {}
key_account = credentials_list[credentialIndex]
file_dict[key_account] = "null"
dict['origin'] = file_dict
# Make account list for replication
except_credentials_list = []
replica_list = []
for group in receivedCredential:
if credentials_list[credentialIndex] in receivedCredential[group]:
credentials_list_in_group = receivedCredential[group]
remove_used_credential_index = credentials_list_in_group.index(credentials_list[credentialIndex])
replica_list = credentials_list_in_group.copy()
replica_list.pop(remove_used_credential_index)
else:
except_credentials_list = except_credentials_list + receivedCredential[group]
if len(replica_list) < 1:
if addGroupIdx < len(except_credentials_list)-1:
replica_list.append(except_credentials_list[addGroupIdx])
addGroupIdx += 1
replica_list.append(except_credentials_list[addGroupIdx])
addGroupIdx += 1
else:
addGroupIdx = 0
elif len(replica_list) == 1:
if addGroupIdx < len(except_credentials_list):
replica_list.append(except_credentials_list[addGroupIdx])
addGroupIdx += 1
else:
addGroupIdx = 0
# Input account replication list
replica_dict = {}
if circle % 2 == 0:
replica_dict[replica_list.pop()] = "null"
replica_dict[replica_list.pop()] = "null"
dict['replication'] = replica_dict
else:
replica_dict[replica_list.pop(0)] = "null"
replica_dict[replica_list.pop(0)] = "null"
dict['replication'] = replica_dict
# Input used credential list
for account in dict['origin']:
if account not in used_origin_credential:
used_origin_credential.append(account)
for account in dict['replication']:
if account not in used_replication_credential:
used_replication_credential.append(account)
# Input dictionary in List
chunkInfoList.append(dict)
indexOfChunk += 1
print ("[SYSTEM] Finish split file - Created chunk file '{0}'\n".format(indexOfChunk))
return chunkInfoList, used_origin_credential, used_replication_credential
else:
if os.path.isdir(inputFilePath):
print ("\n[SYSTEM] Input File is directory, Please make zip")
zip = zipfile.ZipFile(fileName+".zip", 'w')
zip_dir("./"+inputFilePath[indexOfSlash+1:], zip)
zip.close()
print ("\n[CREATE] Create a zip archive of a directory")
uploadFile(fileName+".zip")
sys.exit(0)
else:
print ("\n[ERROR ] Input the wrong file path")
sys.exit(0)
def updateFileID():
if os.path.isfile("metadata.json"):
f = open("metadata.json", "r")
readJSON = json.load(f)
f.close()
oldAccountList = []
newAccountList = []
originAccountList = []
replicaAccountList = []
receivedCredential = googleDrive.get_credentials_list()
if (len(readJSON) != 0):
for item in range(0, len(readJSON)):
for key_account in readJSON[item]['origin']:
if readJSON[item]['origin'][key_account] == "null":
if key_account in originAccountList:
originAccountList.append(key_account)
for key_account in readJSON[item]['replication']:
if readJSON[item]['replication'][key_account] == "null":
if key_account in replicaAccountList:
replicaAccountList.append(key_account)
if len(originAccountList) != 0:
for i in range(0, len(originAccountList)):
group = googleDrive.get_group_name_of_credential(originAccountList[i])
if len(group) < 3: # if doesn't exist than print "no_user_id"
newAccountList.extend(receivedCredential[group])
for i in range(0, len(newAccountList)):
if newAccountList[i] in oldAccountList:
del newAccountList[i]
for i in range(len(newAccountList)):
metadata_of_id = googleDrive.get_file_id_and_name_in_shared_folder(originAccountList[i])
key_list = list(metadata_of_id)
else:
print ("[ERROR ] Recovering google drive - {0}".format(originAccountList[i]))
elif len(replicaAccountList) != 0:
for i in range(0, len(replicaAccountList)):
group = googleDrive.get_group_name_of_credential(replicaAccountList[i])
if len(group) < 3: # if doesn't exist than print "no_user_id"
newAccountList.extend(receivedCredential[group])
for i in range(0, len(newAccountList)):
if newAccountList[i] in oldAccountList:
del newAccountList[i]
for i in range(len(newAccountList)):
metadata_of_id = googleDrive.get_file_id_and_name_in_shared_folder(newAccountList[i])
key_list = list(metadata_of_id)
else:
print ("[ERROR ] Recovering... google drive - {0}".format(replicaAccountList[i]))
while len(newAccountList) != 0:
id = newAccountList.pop()
metadata_of_id = googleDrive.get_file_id_and_name_in_shared_folder(id)
key_list = list(metadata_of_id.keys())
for i in range(0, len(readJSON)):
if readJSON[item]['chunkName'] == key_list.pop():
key = readJSON[item]['chunkName']
readJSON[item]['origin'][id] = metadata_of_id[i].keys()
print ("[SYSTEM] Update damaged file ID")
else:
print("[SYSTEM] --------------- You didn't upload file on google drive ---------------")
else:
print("[ERROR ] Doesn't exist metadata.json")