def downloadMetaData(self, simulateDownload=False):
""" Download metadata """
#used for test
if simulateDownload is True:
time.sleep(constConfig.TEST_THREAD_SLEEP_TIME)
return downStatus
self.d.downTime = timezone.now()
# get root directory
root = self.service.metadata("/",
include_deleted=True,
include_media_info=True)
# get others directory
fileMetaData = self.recurseDropTree(root, 5)
self.metadata = fileMetaData
meta = base64.b64encode(json.dumps(fileMetaData))
metaTime = timezone.now()
metaHash = crypto.rsaSignatureSHA256(
meta + crypto.HASH_SEPARATOR + format(metaTime, "U"),
settings.PRIV_KEY)
#store the data
storeFM = FileMetadata(metadata=meta,
tokenID=self.t,
metaTime=metaTime,
metadataHash=metaHash)
storeFM.save()
self.d.threadStatus = constConfig.THREAD_DOWN_META
self.d.save()
def downloadMetaData(self,simulateDownload = False):
""" Download metadata """
#used for test
if simulateDownload is True:
time.sleep(constConfig.TEST_THREAD_SLEEP_TIME)
return downStatus
self.d.downTime = timezone.now()
# get root directory
root = self.service.metadata("/",include_deleted=True,include_media_info=True)
# get others directory
fileMetaData = self.recurseDropTree(root,5)
self.metadata = fileMetaData
meta = base64.b64encode(json.dumps(fileMetaData))
metaTime = timezone.now()
metaHash = crypto.rsaSignatureSHA256(meta+crypto.HASH_SEPARATOR+format(metaTime,"U"),settings.PRIV_KEY)
#store the data
storeFM = FileMetadata(metadata=meta,tokenID=self.t,metaTime=metaTime,metadataHash=metaHash)
storeFM.save()
self.d.threadStatus = constConfig.THREAD_DOWN_META
self.d.save()
def downloadFiles(self,simulateDownload = False): """ Download file with google drive """ #used for tests if simulateDownload is True: #simulate the download by waiting 10 seconds time.sleep(constConfig.TEST_THREAD_SLEEP_TIME) return downStatus #get download folder downDirFullSub = os.path.join(self.downloadDir,constConfig.DOWNLOAD_FILES_FOLDER) if not os.path.isdir(downDirFullSub): os.mkdir(downDirFullSub) #iterate over file and write to disk for item in self.metadata: if 'downloadUrl' in item: url = item['downloadUrl'] elif 'exportLinks' in item: url = item['exportLinks']["application/pdf"] else: url = None if url != None: fileDb = None try: resp, content = self.service._http.request(url) if resp.status == 200: hashFileName = crypto.sha256(item['title']+crypto.HASH_SEPARATOR+item['id']) fullName = os.path.join(downDirFullSub,hashFileName.hexdigest()+ "_" + item['id']) with open(fullName,"wb+") as f: f.write(content) #compute hash h = crypto.rsaSignatureSHA256(fullName,settings.PRIV_KEY,True) else: h = "-" fileDb = FileDownload(fileName=item['title'],alternateName=item['id'],status=resp.status,tokenID=self.t,fileHash=h) fileDb.save() except errors.HttpError, e: #store this entry with the exception code fileDb = FileDownload(fileName=item['item'],alternateName=item['id'],status=e.resp.status,tokenID=self.t,fileHash="-") fileDb.save()
def downloadFiles(self, simulateDownload=False):
""" Download files """
#used for test
if simulateDownload is True:
time.sleep(constConfig.TEST_THREAD_SLEEP_TIME)
return downStatus
downDirFullSub = os.path.join(self.downloadDir,
constConfig.DOWNLOAD_FILES_FOLDER)
if not os.path.isdir(downDirFullSub):
os.mkdir(downDirFullSub)
#for each folder
for c in self.metadata:
#for each file in folder
for f in c['contents']:
if not f['is_dir']: # if is a file
#compute the alternateName
altName = dropboxAlternateName(f['path'], f['modified'])
bName = os.path.basename(f['path'])
try:
with self.service.get_file(f['path']) as f:
hashName = crypto.sha256(bName +
crypto.HASH_SEPARATOR +
altName).hexdigest()
fullPath = os.path.join(downDirFullSub,
hashName + "_" + altName)
outF = open(fullPath, "wb+")
outF.write(f.read())
outF.close()
h = crypto.rsaSignatureSHA256(
fullPath, settings.PRIV_KEY, True)
fDb = FileDownload(fileName=bName,
alternateName=altName,
status=1,
tokenID=self.t,
fileHash=h)
fDb.save()
except dropbox.rest.ErrorResponse as e:
f = FileDownload(fileName=bName,
alternateName=altName,
status=e.status,
tokenID=self.t,
fileHash="-")
f.save()
def createZIPtoVerify(self):
""" Verify a ZIP by computing its signature """
#the download has completed and there is not another ZIP verified
if self.download.threadStatus == constConfig.THREAD_DOWN_FH and self.download.verificationZIP == False:
dstPath = self.createZIP()
#compute the rsa signature
signature = crypto.rsaSignatureSHA256(dstPath,settings.PRIV_KEY,True)
self.download.verificationZIP = True
self.download.verificationZIPSignatureHash = crypto.sha256(signature).hexdigest()
self.download.verificationZIPSignature = signature
self.download.save()
return True
else:
raise Exception("Download not complete or ZIP already exists.")
def downloadMetaData(self,simulateDownload = False):
""" Download the metadata """
#used for tests
if simulateDownload is True:
#simulate the download by waiting 10 seconds
time.sleep(constConfig.TEST_THREAD_SLEEP_TIME)
return downStatus
#download
self.d.downTime = timezone.now()
result = []
page_token = None
while True:
param = {}
if page_token:
param['pageToken'] = page_token
param['maxResults'] = 500
files = self.service.files().list(**param).execute()
result.extend(files['items'])
page_token = files.get('nextPageToken')
if not page_token:
break
self.metadata = result
meta = base64.b64encode(json.dumps(self.metadata))
metaTime = timezone.now()
txt = meta+crypto.HASH_SEPARATOR+format(metaTime,"U")
metaHash = crypto.rsaSignatureSHA256(txt,settings.PRIV_KEY)
storeFM = FileMetadata(metadata=meta,tokenID=self.t,metaTime=metaTime,metadataHash=metaHash)
storeFM.save()
self.d.threadStatus = constConfig.THREAD_DOWN_META
self.d.save()
def createZIPtoVerify(self):
""" Verify a ZIP by computing its signature """
#the download has completed and there is not another ZIP verified
if self.download.threadStatus == constConfig.THREAD_DOWN_FH and self.download.verificationZIP == False:
dstPath = self.createZIP()
#compute the rsa signature
signature = crypto.rsaSignatureSHA256(dstPath, settings.PRIV_KEY,
True)
self.download.verificationZIP = True
self.download.verificationZIPSignatureHash = crypto.sha256(
signature).hexdigest()
self.download.verificationZIPSignature = signature
self.download.save()
return True
else:
raise Exception("Download not complete or ZIP already exists.")
def verifyMetadata(self):
""" Verifiy the file metadata """
hList = list()
meta = FileMetadata.objects.get(tokenID=self.t)
metaFile = meta.metadata
mTime = getTimestamp(meta.metaTime)
#compute hash
h = crypto.sha256(metaFile+crypto.HASH_SEPARATOR+mTime)
#verify
verification = crypto.verifyRSAsignatureSHA256(h,meta.metadataHash,settings.PUB_KEY)
sig = crypto.rsaSignatureSHA256(metaFile+crypto.HASH_SEPARATOR+mTime,settings.PRIV_KEY)
#metadata hash
mSig = crypto.sha256(meta.metadataHash).hexdigest()
return ({'metaID': meta.id,'verificationResult': verification,'mSig':mSig})
def downloadFiles(self,simulateDownload = False): """ Download files """ #used for test if simulateDownload is True: time.sleep(constConfig.TEST_THREAD_SLEEP_TIME) return downStatus downDirFullSub = os.path.join(self.downloadDir, constConfig.DOWNLOAD_FILES_FOLDER) if not os.path.isdir(downDirFullSub): os.mkdir(downDirFullSub) #for each folder for c in self.metadata: #for each file in folder for f in c['contents']: if not f['is_dir']: # if is a file #compute the alternateName altName = dropboxAlternateName(f['path'],f['modified']) bName = os.path.basename(f['path']) try: with self.service.get_file(f['path']) as f: hashName = crypto.sha256(bName+crypto.HASH_SEPARATOR+altName).hexdigest() fullPath = os.path.join(downDirFullSub,hashName+"_"+altName) outF = open(fullPath,"wb+") outF.write(f.read()) outF.close() h = crypto.rsaSignatureSHA256(fullPath,settings.PRIV_KEY,True) fDb = FileDownload(fileName=bName,alternateName=altName,status=1,tokenID=self.t,fileHash=h) fDb.save() except dropbox.rest.ErrorResponse as e: f = FileDownload(fileName=bName,alternateName=altName,status=e.status,tokenID=self.t,fileHash="-") f.save()
def downloadHistory(self,simulateDownload = False): """ Download the history for a file """ #used for tests if simulateDownload is True: #simulate the download by waiting 10 seconds time.sleep(constConfig.TEST_THREAD_SLEEP_TIME) return downStatus downDirHistory = os.path.join(self.downloadDir,constConfig.DOWNLOAD_HISTORY_FOLDER) if not os.path.isdir(downDirHistory): os.mkdir(downDirHistory) for item in self.metadata: #folders do not support revision if item['mimeType'] != 'application/vnd.google-apps.folder': fh = None try: fileDownload = FileDownload.objects.get(fileName=item['title'],alternateName=item['id'],tokenID=self.t,status=200) try: #get revisions for this file revs = self.service.revisions().list(fileId=item['id']).execute() if len(revs['items']) > 1: #create a folder for this file revPath = os.path.join(downDirHistory,item['id']) if not os.path.isdir(revPath): os.mkdir(revPath) for r in revs['items']: if 'exportLinks' in r: url = r['exportLinks']["application/pdf"] elif 'downloadUrl' in r: url = r['downloadUrl'] else: url = None if url != None: resp, content = self.service._http.request(url) revItem = base64.b64encode(json.dumps(r)) revID = r['id'] hashFileName = crypto.sha256(item['title']+crypto.HASH_SEPARATOR+revID) downloadTime = timezone.now() revisionMetadataHash = crypto.rsaSignatureSHA256( revItem+crypto.HASH_SEPARATOR+getTimestamp(downloadTime), settings.PRIV_KEY) #if the response is affirmative store the file if resp.status == 200: fullName = os.path.join(revPath,hashFileName.hexdigest()+"_"+revID) with open(fullName,"wb+") as f: f.write(content) # compute hash fileRevisionHash = crypto.rsaSignatureSHA256(fullName,settings.PRIV_KEY,True) else: fileRevisionHash = "-" fh = FileHistory( revision=revID, status=resp.status, fileDownloadID=fileDownload, revisionMetadata=revItem, downloadTime=downloadTime, fileRevisionHash=fileRevisionHash, revisionMetadataHash=revisionMetadataHash ) fh.save() # HTTP when requesting history list or downloading file except errors.HttpError, e: fh = FileHistory(revision="-",status=e.resp.status,fileDownloadID=fileDownload,revisionMetadata="-",fileRevisionHash="-",revisionMetadataHash="-") fh.save() #we cannot download the history for a file that has not been downloaded correctly except ObjectDoesNotExist as e: print e pass
def downloadHistory(self, simulateDownload=False):
""" Download the history for dropbox """
#used for test
if simulateDownload is True:
time.sleep(constConfig.TEST_THREAD_SLEEP_TIME)
return downStatus
downDirFullSub = os.path.join(self.downloadDir,
constConfig.DOWNLOAD_HISTORY_FOLDER)
if not os.path.isdir(downDirFullSub):
os.mkdir(downDirFullSub)
for c in self.metadata:
#for each file in folder
for f in c['contents']:
if not f['is_dir']:
rev = self.service.revisions(f['path'])
if len(rev) > 1: # one revision means original file
#compute alternate name for db lookup, from the first revision that is the original file
modified = rev[0]['modified']
path = rev[0]['path']
s = path.encode('utf-8') + modified.encode('utf-8')
altName = md5.new(s).hexdigest()
bName = os.path.basename(path)
#get file download id
try:
fDown = FileDownload.objects.get(
fileName=bName,
alternateName=altName,
tokenID=self.t,
status=1)
del rev[0]
# create a directory to store file revision
revPath = os.path.join(downDirFullSub, altName)
if not os.path.isdir(revPath):
os.mkdir(revPath)
for r in rev:
revID = r['rev']
hashName = crypto.sha256(
bName + crypto.HASH_SEPARATOR +
revID).hexdigest()
fullPath = os.path.join(
revPath, hashName + "_" + revID)
try:
#get revision
with self.service.get_file(
f['path'], revID) as revF:
outF = open(fullPath, "wb+")
outF.write(revF.read())
outF.close()
#hash
rEnc = base64.b64encode(json.dumps(r))
downloadTime = timezone.now()
fileRevisionHash = crypto.rsaSignatureSHA256(
fullPath, settings.PRIV_KEY, True)
revisionMetadataHash = crypto.rsaSignatureSHA256(
rEnc + crypto.HASH_SEPARATOR +
format(downloadTime, "U"),
settings.PRIV_KEY)
fDb = FileHistory(
revision=revID,
status=1,
fileDownloadID=fDown,
revisionMetadata=rEnc,
downloadTime=downloadTime,
revisionMetadataHash=
revisionMetadataHash,
fileRevisionHash=fileRevisionHash)
fDb.save()
#when there is an error with the download of the history
except dropbox.rest.ErrorResponse as e:
fDb = FileHistory(revision=revID,
status=e.status,
fileDownloadID=fDown,
revisionMetadata="-",
fileRevisionHash="-",
revisionMetadataHash="-")
fDb.save()
#we cannot download the history of a file that has not been downloaded correctly
except ObjectDoesNotExist as e:
print e
pass
def downloadHistory(self,simulateDownload = False):
""" Download the history for dropbox """
#used for test
if simulateDownload is True:
time.sleep(constConfig.TEST_THREAD_SLEEP_TIME)
return downStatus
downDirFullSub = os.path.join(self.downloadDir,constConfig.DOWNLOAD_HISTORY_FOLDER)
if not os.path.isdir(downDirFullSub):
os.mkdir(downDirFullSub)
for c in self.metadata:
#for each file in folder
for f in c['contents']:
if not f['is_dir']:
rev = self.service.revisions(f['path'])
if len(rev) > 1: # one revision means original file
#compute alternate name for db lookup, from the first revision that is the original file
modified = rev[0]['modified']
path = rev[0]['path']
s = path.encode('utf-8') + modified.encode('utf-8')
altName = md5.new(s).hexdigest()
bName = os.path.basename(path)
#get file download id
try:
fDown = FileDownload.objects.get(fileName=bName,alternateName=altName,tokenID=self.t,status=1)
del rev[0]
# create a directory to store file revision
revPath = os.path.join(downDirFullSub,altName)
if not os.path.isdir(revPath):
os.mkdir(revPath)
for r in rev:
revID = r['rev']
hashName = crypto.sha256(bName+crypto.HASH_SEPARATOR+revID).hexdigest()
fullPath = os.path.join(revPath,hashName+"_"+revID)
try:
#get revision
with self.service.get_file(f['path'],revID) as revF:
outF = open(fullPath,"wb+")
outF.write(revF.read())
outF.close()
#hash
rEnc = base64.b64encode(json.dumps(r))
downloadTime = timezone.now()
fileRevisionHash = crypto.rsaSignatureSHA256(fullPath,settings.PRIV_KEY,True)
revisionMetadataHash = crypto.rsaSignatureSHA256(rEnc+crypto.HASH_SEPARATOR+format(downloadTime,"U"),settings.PRIV_KEY)
fDb = FileHistory(
revision=revID,
status=1,
fileDownloadID=fDown,
revisionMetadata=rEnc,
downloadTime=downloadTime,
revisionMetadataHash=revisionMetadataHash,
fileRevisionHash=fileRevisionHash
)
fDb.save()
#when there is an error with the download of the history
except dropbox.rest.ErrorResponse as e:
fDb = FileHistory(revision=revID,status=e.status,fileDownloadID=fDown,revisionMetadata="-",fileRevisionHash="-",revisionMetadataHash="-")
fDb.save()
#we cannot download the history of a file that has not been downloaded correctly
except ObjectDoesNotExist as e:
print e
pass