def verifyFileDownload(self,resType): """ Verify the files of a token """ hList = list() downloadFolder = self.d.folder for f in FileDownload.objects.filter(tokenID=self.t): hashName = crypto.sha256(f.fileName+crypto.HASH_SEPARATOR+f.alternateName).hexdigest() path = os.path.join(self.filesFolder,hashName+"_"+f.alternateName) isFile = os.path.isfile(path) if isFile: #first compute the hash h = crypto.sha256File(path) #now verify the hash sourceSignature = f.fileHash verification = crypto.verifyRSAsignatureSHA256(h,sourceSignature,settings.PUB_KEY) #history only if specified if resType == constConfig.VERIFY_CHOICE_FILESHISTORY: historyVerification = self.verifyHistory(f) else: historyVerification = None hList.append({'fID': f.id,'fName':f.fileName,'fSig':crypto.sha256(f.fileHash).hexdigest(),'verificationResult':verification,'history':historyVerification}) elif not isFile: hList.append({'fID': f.id,'fName':f.fileName,'fSig':"File does not exists on disk",'verificationResult':-1,'history': list()}) return hList
def verifyHistory(self,fileDownload): """ Verify file history """ hList = list() for fh in FileHistory.objects.filter(fileDownloadID=fileDownload): #verification of revision metadata revMeta = fh.revisionMetadata revDownTime = getTimestamp(fh.downloadTime) h = crypto.sha256(revMeta+crypto.HASH_SEPARATOR+revDownTime) verification = crypto.verifyRSAsignatureSHA256(h,fh.revisionMetadataHash,settings.PUB_KEY) hashName = crypto.sha256(fileDownload.fileName+crypto.HASH_SEPARATOR+fh.revision).hexdigest() path = os.path.join(self.historyFolder,fileDownload.alternateName,hashName+"_"+fh.revision) isPath = os.path.isfile(path) if isPath: #verification of file history fHash = crypto.sha256File(path) verificationFile = crypto.verifyRSAsignatureSHA256(fHash,fh.fileRevisionHash,settings.PUB_KEY) hList.append({'hID': fh.id,'revID':fh.revision,'metadataVerificationResult': verification,'fileVerificationResult':verificationFile}) else: hList.append({'hID': fh.id,'revID':fh.revision,'metadataVerificationResult': verification,'fileVerificationResult':"File does not exists on disk"}) return hList
def displaySingle(self, title, altName, downAltName, fileFolder): """ Display a single file """ if fileFolder == "file" or altName == downAltName: f = constConfig.DOWNLOAD_FILES_FOLDER elif fileFolder == "history": f = os.path.join(constConfig.DOWNLOAD_HISTORY_FOLDER, downAltName) name = crypto.sha256(title + crypto.HASH_SEPARATOR + altName).hexdigest() + "_" + altName downloadFolder = Download.objects.get( tokenID=self.t, threadStatus=constConfig.THREAD_TS).folder fullPath = os.path.join(settings.DOWNLOAD_DIR, downloadFolder, f, name) print os.path.isfile(fullPath) print fullPath mime = magic.Magic(mime=True) mimeType = mime.from_file(fullPath) if mimeType in constConfig.ALLOWED_MIME_TYPE[1:-1]: t = Thubmnailer() t.cacheImg(fullPath, os.path.join(settings.DIFF_DIR, name + ".thumbnail")) return {'name': name, 'mime': mimeType} elif mimeType == constConfig.ALLOWED_MIME_TYPE[0]: dest = os.path.join(settings.DIFF_DIR, name) if not os.path.isfile(dest): shutil.copy(fullPath, dest) return {'name': name, 'mime': constConfig.ALLOWED_MIME_TYPE[0]}
def displaySingle(self,title,altName,downAltName,fileFolder): """ Display a single file """ if fileFolder == "file" or altName == downAltName: f = constConfig.DOWNLOAD_FILES_FOLDER elif fileFolder == "history": f = os.path.join(constConfig.DOWNLOAD_HISTORY_FOLDER,downAltName) name = crypto.sha256(title+crypto.HASH_SEPARATOR+altName).hexdigest() + "_" + altName downloadFolder = Download.objects.get(tokenID=self.t,threadStatus = constConfig.THREAD_TS).folder fullPath = os.path.join(settings.DOWNLOAD_DIR, downloadFolder,f,name) print os.path.isfile(fullPath) print fullPath mime = magic.Magic(mime=True) mimeType = mime.from_file(fullPath) if mimeType in constConfig.ALLOWED_MIME_TYPE[1:-1]: t = Thubmnailer() t.cacheImg(fullPath,os.path.join(settings.DIFF_DIR,name+".thumbnail")) return {'name': name,'mime':mimeType} elif mimeType == constConfig.ALLOWED_MIME_TYPE[0]: dest = os.path.join(settings.DIFF_DIR,name) if not os.path.isfile(dest): shutil.copy(fullPath, dest) return {'name': name, 'mime':constConfig.ALLOWED_MIME_TYPE[0]}
def verifyMetadata(self): """ Verifiy the file metadata """ hList = list() meta = FileMetadata.objects.get(tokenID=self.t) metaFile = meta.metadata mTime = getTimestamp(meta.metaTime) #compute hash h = crypto.sha256(metaFile+crypto.HASH_SEPARATOR+mTime) #verify verification = crypto.verifyRSAsignatureSHA256(h,meta.metadataHash,settings.PUB_KEY) sig = crypto.rsaSignatureSHA256(metaFile+crypto.HASH_SEPARATOR+mTime,settings.PRIV_KEY) #metadata hash mSig = crypto.sha256(meta.metadataHash).hexdigest() return ({'metaID': meta.id,'verificationResult': verification,'mSig':mSig})
def compareFromReport(self): """ Compare the file found in the report with the ones downloaded """ res = list() #get the report report = openReport(self.t.cloudItem) if report is not None: cloudFiles = report[2]['objects'] if cloudFiles is not None: #compute the hash of the downloaded files: #get the files of the token files = FileDownload.objects.filter(tokenID=self.t) downFolder = Download.objects.get(tokenID=self.t).folder for f in files: localRes = list() fileName = crypto.sha256(f.fileName + crypto.HASH_SEPARATOR + f.alternateName).hexdigest() basePath = os.path.join(settings.DOWNLOAD_DIR, downFolder, "files") fullPath = os.path.join(basePath, fileName + "_" + f.alternateName) if os.path.isfile(fullPath): fileDigest = crypto.sha256File(fullPath).hexdigest() for cloud in cloudFiles: #check if the CSP of the token is in the report if self.t.serviceType in cloud[ 'cloudService'].lower(): #now check if in the report we have the same file for cloudFile in cloud['files']: if cloudFile['hash'] == fileDigest: localRes.append({ 'csp': cloud['cloudService'], 'files': cloudFile }) res.append({'file': f, 'comparableFiles': localRes}) return res
def downloadFiles(self, simulateDownload=False): """ Download files """ #used for test if simulateDownload is True: time.sleep(constConfig.TEST_THREAD_SLEEP_TIME) return downStatus downDirFullSub = os.path.join(self.downloadDir, constConfig.DOWNLOAD_FILES_FOLDER) if not os.path.isdir(downDirFullSub): os.mkdir(downDirFullSub) #for each folder for c in self.metadata: #for each file in folder for f in c['contents']: if not f['is_dir']: # if is a file #compute the alternateName altName = dropboxAlternateName(f['path'], f['modified']) bName = os.path.basename(f['path']) try: with self.service.get_file(f['path']) as f: hashName = crypto.sha256(bName + crypto.HASH_SEPARATOR + altName).hexdigest() fullPath = os.path.join(downDirFullSub, hashName + "_" + altName) outF = open(fullPath, "wb+") outF.write(f.read()) outF.close() h = crypto.rsaSignatureSHA256( fullPath, settings.PRIV_KEY, True) fDb = FileDownload(fileName=bName, alternateName=altName, status=1, tokenID=self.t, fileHash=h) fDb.save() except dropbox.rest.ErrorResponse as e: f = FileDownload(fileName=bName, alternateName=altName, status=e.status, tokenID=self.t, fileHash="-") f.save()
def downloadFiles(self,simulateDownload = False): """ Download file with google drive """ #used for tests if simulateDownload is True: #simulate the download by waiting 10 seconds time.sleep(constConfig.TEST_THREAD_SLEEP_TIME) return downStatus #get download folder downDirFullSub = os.path.join(self.downloadDir,constConfig.DOWNLOAD_FILES_FOLDER) if not os.path.isdir(downDirFullSub): os.mkdir(downDirFullSub) #iterate over file and write to disk for item in self.metadata: if 'downloadUrl' in item: url = item['downloadUrl'] elif 'exportLinks' in item: url = item['exportLinks']["application/pdf"] else: url = None if url != None: fileDb = None try: resp, content = self.service._http.request(url) if resp.status == 200: hashFileName = crypto.sha256(item['title']+crypto.HASH_SEPARATOR+item['id']) fullName = os.path.join(downDirFullSub,hashFileName.hexdigest()+ "_" + item['id']) with open(fullName,"wb+") as f: f.write(content) #compute hash h = crypto.rsaSignatureSHA256(fullName,settings.PRIV_KEY,True) else: h = "-" fileDb = FileDownload(fileName=item['title'],alternateName=item['id'],status=resp.status,tokenID=self.t,fileHash=h) fileDb.save() except errors.HttpError, e: #store this entry with the exception code fileDb = FileDownload(fileName=item['item'],alternateName=item['id'],status=e.resp.status,tokenID=self.t,fileHash="-") fileDb.save()
def createZIPtoVerify(self): """ Verify a ZIP by computing its signature """ #the download has completed and there is not another ZIP verified if self.download.threadStatus == constConfig.THREAD_DOWN_FH and self.download.verificationZIP == False: dstPath = self.createZIP() #compute the rsa signature signature = crypto.rsaSignatureSHA256(dstPath,settings.PRIV_KEY,True) self.download.verificationZIP = True self.download.verificationZIPSignatureHash = crypto.sha256(signature).hexdigest() self.download.verificationZIPSignature = signature self.download.save() return True else: raise Exception("Download not complete or ZIP already exists.")
def createZIPtoVerify(self): """ Verify a ZIP by computing its signature """ #the download has completed and there is not another ZIP verified if self.download.threadStatus == constConfig.THREAD_DOWN_FH and self.download.verificationZIP == False: dstPath = self.createZIP() #compute the rsa signature signature = crypto.rsaSignatureSHA256(dstPath, settings.PRIV_KEY, True) self.download.verificationZIP = True self.download.verificationZIPSignatureHash = crypto.sha256( signature).hexdigest() self.download.verificationZIPSignature = signature self.download.save() return True else: raise Exception("Download not complete or ZIP already exists.")
def compareFromReport(self): """ Compare the file found in the report with the ones downloaded """ res = list() #get the report report = openReport(self.t.cloudItem) if report is not None: cloudFiles = report[2]['objects'] if cloudFiles is not None: #compute the hash of the downloaded files: #get the files of the token files = FileDownload.objects.filter(tokenID=self.t) downFolder = Download.objects.get(tokenID=self.t).folder for f in files: localRes = list() fileName = crypto.sha256(f.fileName+crypto.HASH_SEPARATOR+f.alternateName).hexdigest() basePath = os.path.join(settings.DOWNLOAD_DIR,downFolder,"files") fullPath = os.path.join(basePath,fileName+"_"+f.alternateName) if os.path.isfile(fullPath): fileDigest = crypto.sha256File(fullPath).hexdigest() for cloud in cloudFiles: #check if the CSP of the token is in the report if self.t.serviceType in cloud['cloudService'].lower(): #now check if in the report we have the same file for cloudFile in cloud['files']: if cloudFile['hash'] == fileDigest: localRes.append({'csp':cloud['cloudService'],'files':cloudFile}) res.append({'file':f,'comparableFiles':localRes}) return res
def __init__(self,cloudItem,browserParam): self.fb = None #check that an import exists imp = Upload.objects.get(cloudItemID=cloudItem) #compute import path importPath = crypto.sha256(imp.fileName+crypto.HASH_SEPARATOR+getTimestamp(imp.uploadDate)).hexdigest() self.filePath = os.path.join(settings.UPLOAD_DIR,str(cloudItem.id),importPath,imp.fileName) profile = str(browserParam.split("_",1)[1]) #windows profile are stored with Profile\in front if profile.startswith("Profile"): profile = profile[9:] if int(browserParam[0]) == constConfig.HISTORY_FORM_CHROME: self.fb = GoogleChromeFiles(self.filePath,profile) elif int(browserParam[0]) == constConfig.HISTORY_FORM_FF: self.fb = FirefoxFiles(self.filePath,profile)
def downloadFiles(self,simulateDownload = False): """ Download files """ #used for test if simulateDownload is True: time.sleep(constConfig.TEST_THREAD_SLEEP_TIME) return downStatus downDirFullSub = os.path.join(self.downloadDir, constConfig.DOWNLOAD_FILES_FOLDER) if not os.path.isdir(downDirFullSub): os.mkdir(downDirFullSub) #for each folder for c in self.metadata: #for each file in folder for f in c['contents']: if not f['is_dir']: # if is a file #compute the alternateName altName = dropboxAlternateName(f['path'],f['modified']) bName = os.path.basename(f['path']) try: with self.service.get_file(f['path']) as f: hashName = crypto.sha256(bName+crypto.HASH_SEPARATOR+altName).hexdigest() fullPath = os.path.join(downDirFullSub,hashName+"_"+altName) outF = open(fullPath,"wb+") outF.write(f.read()) outF.close() h = crypto.rsaSignatureSHA256(fullPath,settings.PRIV_KEY,True) fDb = FileDownload(fileName=bName,alternateName=altName,status=1,tokenID=self.t,fileHash=h) fDb.save() except dropbox.rest.ErrorResponse as e: f = FileDownload(fileName=bName,alternateName=altName,status=e.status,tokenID=self.t,fileHash="-") f.save()
def downloadHistory(self,simulateDownload = False): """ Download the history for a file """ #used for tests if simulateDownload is True: #simulate the download by waiting 10 seconds time.sleep(constConfig.TEST_THREAD_SLEEP_TIME) return downStatus downDirHistory = os.path.join(self.downloadDir,constConfig.DOWNLOAD_HISTORY_FOLDER) if not os.path.isdir(downDirHistory): os.mkdir(downDirHistory) for item in self.metadata: #folders do not support revision if item['mimeType'] != 'application/vnd.google-apps.folder': fh = None try: fileDownload = FileDownload.objects.get(fileName=item['title'],alternateName=item['id'],tokenID=self.t,status=200) try: #get revisions for this file revs = self.service.revisions().list(fileId=item['id']).execute() if len(revs['items']) > 1: #create a folder for this file revPath = os.path.join(downDirHistory,item['id']) if not os.path.isdir(revPath): os.mkdir(revPath) for r in revs['items']: if 'exportLinks' in r: url = r['exportLinks']["application/pdf"] elif 'downloadUrl' in r: url = r['downloadUrl'] else: url = None if url != None: resp, content = self.service._http.request(url) revItem = base64.b64encode(json.dumps(r)) revID = r['id'] hashFileName = crypto.sha256(item['title']+crypto.HASH_SEPARATOR+revID) downloadTime = timezone.now() revisionMetadataHash = crypto.rsaSignatureSHA256( revItem+crypto.HASH_SEPARATOR+getTimestamp(downloadTime), settings.PRIV_KEY) #if the response is affirmative store the file if resp.status == 200: fullName = os.path.join(revPath,hashFileName.hexdigest()+"_"+revID) with open(fullName,"wb+") as f: f.write(content) # compute hash fileRevisionHash = crypto.rsaSignatureSHA256(fullName,settings.PRIV_KEY,True) else: fileRevisionHash = "-" fh = FileHistory( revision=revID, status=resp.status, fileDownloadID=fileDownload, revisionMetadata=revItem, downloadTime=downloadTime, fileRevisionHash=fileRevisionHash, revisionMetadataHash=revisionMetadataHash ) fh.save() # HTTP when requesting history list or downloading file except errors.HttpError, e: fh = FileHistory(revision="-",status=e.resp.status,fileDownloadID=fileDownload,revisionMetadata="-",fileRevisionHash="-",revisionMetadataHash="-") fh.save() #we cannot download the history for a file that has not been downloaded correctly except ObjectDoesNotExist as e: print e pass
def searchMetaData(request,form,tokenID,cloudItem,start): """ Make a search through the metadata """ dajax = Dajax() try: t = parseAjaxParam(tokenID) ciChk = checkCloudItem(cloudItem,request.user.id) tknObj = checkAccessToken(t,ciChk) searchStep = 100 f = MetaSearch(deserialize_form(form)) if f.is_valid(): startResTime = time.time() #compute hash of the search form for the cache searchHash = crypto.sha256(form).hexdigest() """searchHash = crypto.sha256(f.cleaned_data['formType'][0]+crypto.HASH_SEPARATOR+ f.cleaned_data['email']+crypto.HASH_SEPARATOR+ f.cleaned_data['filename']+crypto.HASH_SEPARATOR+ f.cleaned_data['givenname']+crypto.HASH_SEPARATOR+ f.cleaned_data['resType'][0]+crypto.HASH_SEPARATOR+ f.cleaned_data['mimeType']+crypto.HASH_SEPARATOR+ str(f.cleaned_data['startDate'])+crypto.HASH_SEPARATOR+ str(f.cleaned_data['endDate']) ).hexdigest()""" if "searchCache" in request.session and request.session['searchCacheID'] == searchHash: res = json.loads(request.session["searchCache"]) else: mc = MetadataController(tknObj) res = mc.metadataSearch( int(f.cleaned_data['formType'][0]), f.cleaned_data['email'], f.cleaned_data['filename'], f.cleaned_data['givenname'], int(f.cleaned_data['resType'][0]), int(f.cleaned_data['mimeType']), f.cleaned_data['startDate'], f.cleaned_data['endDate'] ) request.session["searchCacheID"] = searchHash request.session["searchCache"] = json.dumps(res) #computation for pager totalPages = int(math.ceil(float(len(res))/100.0)) resultsSlice = res[start:(start+searchStep)] stopResTime = time.time() parsedTable = render_to_string("dashboard/cloudservice/searchTable.html", {'data': resultsSlice,'totalPages':range(totalPages),'totalRes':len(res),'resTime': stopResTime-startResTime,'platform':tknObj.serviceType}) dajax.assign("#searchRes","innerHTML",parsedTable) dajax.assign("#searchError","innerHTML","") dajax.remove_css_class("#searchError",['alert','alert-danger']) else: dajax.assign("#searchError","innerHTML","Please fill all fields") dajax.add_css_class("#searchError",['alert','alert-danger']) except Exception as e: dajax.assign("#searchError","innerHTML",formatException(e)) dajax.add_css_class("#searchError",['alert','alert-danger']) return dajax.json()
def fileInfo(self): """ Displays information about the downloaded files """ history = [] rowElem = list() self.story.append(Paragraph("Downloaded Files",styles['Heading1'])) #files files = DbInterface.getAllFileDownload(self.t) data = [[Paragraph("<b>ID</b>",self.normalStyle), Paragraph("<b>File Name</b>",self.normalStyle), Paragraph("<b>Download Time</b>",self.normalStyle), Paragraph("<b>Signature</b>",self.normalStyle)]] for f in files: line = [Paragraph(str(f.id),self.normalStyle), Paragraph(f.fileName,self.wrappedStyle), Paragraph(str(timezone.localtime(f.downloadTime)),self.wrappedStyle), Paragraph(crypto.sha256(f.fileHash).hexdigest(),self.wrappedStyle) ] data.append(line) hist = DbInterface.getHistoryForFile(f) #if we have an history if len(hist) > 0: #history.append([f.id,DbInterface.getHistoryForFile(f)]) histData = [[Paragraph("<b>ID</b>",self.normalStyle), Paragraph("<b>Revision</b>",self.normalStyle), Paragraph("<b>Download Time</b>",self.normalStyle), Paragraph("<b>Signature</b>",self.normalStyle) ]] for h in hist: histData.append([ Paragraph(str(h.id),self.normalStyle), Paragraph(h.revision,self.normalStyle), Paragraph(str(timezone.localtime(h.downloadTime)),self.normalStyle), Paragraph(crypto.sha256(h.fileRevisionHash).hexdigest(),self.normalStyle) ]) histT = Table(histData, colWidths=(1.5*cm,6*cm,6*cm,6*cm)) histT.setStyle(TableStyle([('INNERGRID', (0,0), (-1,-1), 0.25, colors.black), ('BOX', (0,0), (-1,-1), 0.25, colors.black),])) data.append(["",histT,"",""]) #row to which apply the colspan rowElem.append(('BACKGROUND',(0,len(data)-1),(0,len(data)-1),colors.green)) rowElem.append(('SPAN',(1,len(data)-1),(3,len(data)-1))) t = Table(data, colWidths=(2*cm,9*cm,5*cm,8.5*cm)) styleParam = [('INNERGRID', (0,0), (-1,-1), 0.25, colors.black),('BOX', (0,0), (-1,-1), 0.25, colors.black),] styleParam.extend(rowElem) print rowElem print styleParam ts = TableStyle(styleParam) t.setStyle(ts) self.story.append(t) self.story.append(PageBreak())
def compareTwo(self, revOneID, revTwoID, altName): """ Compare two revision of the same file and check for diff """ finalDiffName = None downloadFolder = Download.objects.get( tokenID=self.t, threadStatus=constConfig.THREAD_TS).folder diffFile = FileDownload.objects.get(tokenID=self.t, alternateName=altName) #diff file full path diffPath = os.path.join(settings.DOWNLOAD_DIR, downloadFolder, constConfig.DOWNLOAD_HISTORY_FOLDER, diffFile.alternateName) #get the two file nameOne = crypto.sha256(diffFile.fileName + crypto.HASH_SEPARATOR + revOneID).hexdigest() nameTwo = crypto.sha256(diffFile.fileName + crypto.HASH_SEPARATOR + revTwoID).hexdigest() revOnePath = os.path.join(diffPath, nameOne + "_" + revOneID) revTwoPath = os.path.join(diffPath, nameTwo + "_" + revTwoID) #check if dropbox that one of the file is not the original if self.t.serviceType == constConfig.CSP_DROPBOX: #try to get the revision from the filedownload table if diffFile.alternateName == revOneID: assumeNameOne = crypto.sha256( diffFile.fileName + crypto.HASH_SEPARATOR + diffFile.alternateName).hexdigest() assumedPath = os.path.join( settings.DOWNLOAD_DIR, downloadFolder, constConfig.DOWNLOAD_FILES_FOLDER, assumeNameOne + "_" + diffFile.alternateName) #overwrite revOnePath only if the path exists in the files folder, otherwise is in the deleted folder if os.path.isfile(assumedPath): revOnePath = assumedPath elif diffFile.alternateName == revTwoID: assumeNameTwo = crypto.sha256( diffFile.fileName + crypto.HASH_SEPARATOR + diffFile.alternateName).hexdigest() assumedPath = os.path.join( settings.DOWNLOAD_DIR, downloadFolder, constConfig.DOWNLOAD_FILES_FOLDER, assumeNameTwo + "_" + diffFile.alternateName) if os.path.isfile(assumedPath): revTwoPath = assumedPath #check that the two path actually exists. This because the actual file in dropbox (rightmost in the file history time line) does not exists if it has been deleted. So we will have an entry in the timeline for a version of a file that does not exist. if not os.path.isfile(revOnePath) or not os.path.isfile(revTwoPath): raise Exception( "One of the two file does not exist. (Is this a deleted file on Dropbox?)" ) print revOnePath print revTwoPath #check allowed mime mime = magic.Magic(mime=True) mimeOne = mime.from_file(revOnePath) mimeTwo = mime.from_file(revTwoPath) if mimeOne not in constConfig.ALLOWED_MIME_TYPE or mimeTwo not in constConfig.ALLOWED_MIME_TYPE: raise Exception("File type not supported") data = None mimeList = list() #pdf files if mimeOne == mimeTwo == constConfig.ALLOWED_MIME_TYPE[0]: resultDiffName = "diff_" + str( self.t.id) + "_" + revOneID + "_" + revTwoID + ".pdf" diffName = self.pdfDiff(revOnePath, revTwoPath, diffPath, resultDiffName) data = {"diffName": diffName} mimeList.append(constConfig.ALLOWED_MIME_TYPE[0]) #images elif mimeOne and mimeTwo in constConfig.ALLOWED_MIME_TYPE[1:-1]: hash1, hash2 = self.imgDiff(revOnePath, revTwoPath) mimeList.append(mime.from_file(revOnePath)) mimeList.append(mime.from_file(revTwoPath)) #build data dictionary data = { 'hash1': hash1, 'hash2': hash2, 'file1': diffFile.fileName + "_" + revOneID, 'file2': diffFile.fileName + "_" + revTwoID } return {"filename": diffFile.fileName, "mime": mimeList, "data": data}
def compareTwo(self,revOneID,revTwoID,altName): """ Compare two revision of the same file and check for diff """ finalDiffName = None downloadFolder = Download.objects.get(tokenID=self.t,threadStatus = constConfig.THREAD_TS).folder diffFile = FileDownload.objects.get(tokenID=self.t,alternateName=altName) #diff file full path diffPath = os.path.join(settings.DOWNLOAD_DIR,downloadFolder,constConfig.DOWNLOAD_HISTORY_FOLDER,diffFile.alternateName) #get the two file nameOne = crypto.sha256(diffFile.fileName+crypto.HASH_SEPARATOR+revOneID).hexdigest() nameTwo = crypto.sha256(diffFile.fileName+crypto.HASH_SEPARATOR+revTwoID).hexdigest() revOnePath = os.path.join(diffPath,nameOne+"_"+revOneID) revTwoPath = os.path.join(diffPath,nameTwo+"_"+revTwoID) #check if dropbox that one of the file is not the original if self.t.serviceType == constConfig.CSP_DROPBOX: #try to get the revision from the filedownload table if diffFile.alternateName == revOneID: assumeNameOne = crypto.sha256(diffFile.fileName+crypto.HASH_SEPARATOR+diffFile.alternateName).hexdigest() assumedPath = os.path.join(settings.DOWNLOAD_DIR,downloadFolder,constConfig.DOWNLOAD_FILES_FOLDER,assumeNameOne+"_"+diffFile.alternateName) #overwrite revOnePath only if the path exists in the files folder, otherwise is in the deleted folder if os.path.isfile(assumedPath): revOnePath = assumedPath elif diffFile.alternateName == revTwoID: assumeNameTwo = crypto.sha256(diffFile.fileName+crypto.HASH_SEPARATOR+diffFile.alternateName).hexdigest() assumedPath = os.path.join(settings.DOWNLOAD_DIR,downloadFolder,constConfig.DOWNLOAD_FILES_FOLDER,assumeNameTwo+"_"+diffFile.alternateName) if os.path.isfile(assumedPath): revTwoPath = assumedPath #check that the two path actually exists. This because the actual file in dropbox (rightmost in the file history time line) does not exists if it has been deleted. So we will have an entry in the timeline for a version of a file that does not exist. if not os.path.isfile(revOnePath) or not os.path.isfile(revTwoPath): raise Exception("One of the two file does not exist. (Is this a deleted file on Dropbox?)") print revOnePath print revTwoPath #check allowed mime mime = magic.Magic(mime=True) mimeOne = mime.from_file(revOnePath) mimeTwo = mime.from_file(revTwoPath) if mimeOne not in constConfig.ALLOWED_MIME_TYPE or mimeTwo not in constConfig.ALLOWED_MIME_TYPE: raise Exception("File type not supported") data = None mimeList = list() #pdf files if mimeOne == mimeTwo == constConfig.ALLOWED_MIME_TYPE[0]: resultDiffName = "diff_"+str(self.t.id)+"_"+revOneID+"_"+revTwoID+".pdf" diffName = self.pdfDiff(revOnePath,revTwoPath,diffPath,resultDiffName) data = {"diffName":diffName} mimeList.append(constConfig.ALLOWED_MIME_TYPE[0]) #images elif mimeOne and mimeTwo in constConfig.ALLOWED_MIME_TYPE[1:-1]: hash1,hash2 = self.imgDiff(revOnePath,revTwoPath) mimeList.append(mime.from_file(revOnePath)) mimeList.append(mime.from_file(revTwoPath)) #build data dictionary data = {'hash1': hash1,'hash2': hash2, 'file1':diffFile.fileName+"_"+revOneID,'file2':diffFile.fileName+"_"+revTwoID} return {"filename": diffFile.fileName,"mime": mimeList, "data": data}
def fileInfo(self): """ Displays information about the downloaded files """ history = [] rowElem = list() self.story.append(Paragraph("Downloaded Files", styles['Heading1'])) #files files = DbInterface.getAllFileDownload(self.t) data = [[ Paragraph("<b>ID</b>", self.normalStyle), Paragraph("<b>File Name</b>", self.normalStyle), Paragraph("<b>Download Time</b>", self.normalStyle), Paragraph("<b>Signature</b>", self.normalStyle) ]] for f in files: line = [ Paragraph(str(f.id), self.normalStyle), Paragraph(f.fileName, self.wrappedStyle), Paragraph(str(timezone.localtime(f.downloadTime)), self.wrappedStyle), Paragraph( crypto.sha256(f.fileHash).hexdigest(), self.wrappedStyle) ] data.append(line) hist = DbInterface.getHistoryForFile(f) #if we have an history if len(hist) > 0: #history.append([f.id,DbInterface.getHistoryForFile(f)]) histData = [[ Paragraph("<b>ID</b>", self.normalStyle), Paragraph("<b>Revision</b>", self.normalStyle), Paragraph("<b>Download Time</b>", self.normalStyle), Paragraph("<b>Signature</b>", self.normalStyle) ]] for h in hist: histData.append([ Paragraph(str(h.id), self.normalStyle), Paragraph(h.revision, self.normalStyle), Paragraph(str(timezone.localtime(h.downloadTime)), self.normalStyle), Paragraph( crypto.sha256(h.fileRevisionHash).hexdigest(), self.normalStyle) ]) histT = Table(histData, colWidths=(1.5 * cm, 6 * cm, 6 * cm, 6 * cm)) histT.setStyle( TableStyle([ ('INNERGRID', (0, 0), (-1, -1), 0.25, colors.black), ('BOX', (0, 0), (-1, -1), 0.25, colors.black), ])) data.append(["", histT, "", ""]) #row to which apply the colspan rowElem.append(('BACKGROUND', (0, len(data) - 1), (0, len(data) - 1), colors.green)) rowElem.append( ('SPAN', (1, len(data) - 1), (3, len(data) - 1))) t = Table(data, colWidths=(2 * cm, 9 * cm, 5 * cm, 8.5 * cm)) styleParam = [ ('INNERGRID', (0, 0), (-1, -1), 0.25, colors.black), ('BOX', (0, 0), (-1, -1), 0.25, colors.black), ] styleParam.extend(rowElem) print rowElem print styleParam ts = TableStyle(styleParam) t.setStyle(ts) self.story.append(t) self.story.append(PageBreak())
def downloadHistory(self, simulateDownload=False): """ Download the history for dropbox """ #used for test if simulateDownload is True: time.sleep(constConfig.TEST_THREAD_SLEEP_TIME) return downStatus downDirFullSub = os.path.join(self.downloadDir, constConfig.DOWNLOAD_HISTORY_FOLDER) if not os.path.isdir(downDirFullSub): os.mkdir(downDirFullSub) for c in self.metadata: #for each file in folder for f in c['contents']: if not f['is_dir']: rev = self.service.revisions(f['path']) if len(rev) > 1: # one revision means original file #compute alternate name for db lookup, from the first revision that is the original file modified = rev[0]['modified'] path = rev[0]['path'] s = path.encode('utf-8') + modified.encode('utf-8') altName = md5.new(s).hexdigest() bName = os.path.basename(path) #get file download id try: fDown = FileDownload.objects.get( fileName=bName, alternateName=altName, tokenID=self.t, status=1) del rev[0] # create a directory to store file revision revPath = os.path.join(downDirFullSub, altName) if not os.path.isdir(revPath): os.mkdir(revPath) for r in rev: revID = r['rev'] hashName = crypto.sha256( bName + crypto.HASH_SEPARATOR + revID).hexdigest() fullPath = os.path.join( revPath, hashName + "_" + revID) try: #get revision with self.service.get_file( f['path'], revID) as revF: outF = open(fullPath, "wb+") outF.write(revF.read()) outF.close() #hash rEnc = base64.b64encode(json.dumps(r)) downloadTime = timezone.now() fileRevisionHash = crypto.rsaSignatureSHA256( fullPath, settings.PRIV_KEY, True) revisionMetadataHash = crypto.rsaSignatureSHA256( rEnc + crypto.HASH_SEPARATOR + format(downloadTime, "U"), settings.PRIV_KEY) fDb = FileHistory( revision=revID, status=1, fileDownloadID=fDown, revisionMetadata=rEnc, downloadTime=downloadTime, revisionMetadataHash= revisionMetadataHash, fileRevisionHash=fileRevisionHash) fDb.save() #when there is an error with the download of the history except dropbox.rest.ErrorResponse as e: fDb = FileHistory(revision=revID, status=e.status, fileDownloadID=fDown, revisionMetadata="-", fileRevisionHash="-", revisionMetadataHash="-") fDb.save() #we cannot download the history of a file that has not been downloaded correctly except ObjectDoesNotExist as e: print e pass
def downloadHistory(self,simulateDownload = False): """ Download the history for dropbox """ #used for test if simulateDownload is True: time.sleep(constConfig.TEST_THREAD_SLEEP_TIME) return downStatus downDirFullSub = os.path.join(self.downloadDir,constConfig.DOWNLOAD_HISTORY_FOLDER) if not os.path.isdir(downDirFullSub): os.mkdir(downDirFullSub) for c in self.metadata: #for each file in folder for f in c['contents']: if not f['is_dir']: rev = self.service.revisions(f['path']) if len(rev) > 1: # one revision means original file #compute alternate name for db lookup, from the first revision that is the original file modified = rev[0]['modified'] path = rev[0]['path'] s = path.encode('utf-8') + modified.encode('utf-8') altName = md5.new(s).hexdigest() bName = os.path.basename(path) #get file download id try: fDown = FileDownload.objects.get(fileName=bName,alternateName=altName,tokenID=self.t,status=1) del rev[0] # create a directory to store file revision revPath = os.path.join(downDirFullSub,altName) if not os.path.isdir(revPath): os.mkdir(revPath) for r in rev: revID = r['rev'] hashName = crypto.sha256(bName+crypto.HASH_SEPARATOR+revID).hexdigest() fullPath = os.path.join(revPath,hashName+"_"+revID) try: #get revision with self.service.get_file(f['path'],revID) as revF: outF = open(fullPath,"wb+") outF.write(revF.read()) outF.close() #hash rEnc = base64.b64encode(json.dumps(r)) downloadTime = timezone.now() fileRevisionHash = crypto.rsaSignatureSHA256(fullPath,settings.PRIV_KEY,True) revisionMetadataHash = crypto.rsaSignatureSHA256(rEnc+crypto.HASH_SEPARATOR+format(downloadTime,"U"),settings.PRIV_KEY) fDb = FileHistory( revision=revID, status=1, fileDownloadID=fDown, revisionMetadata=rEnc, downloadTime=downloadTime, revisionMetadataHash=revisionMetadataHash, fileRevisionHash=fileRevisionHash ) fDb.save() #when there is an error with the download of the history except dropbox.rest.ErrorResponse as e: fDb = FileHistory(revision=revID,status=e.status,fileDownloadID=fDown,revisionMetadata="-",fileRevisionHash="-",revisionMetadataHash="-") fDb.save() #we cannot download the history of a file that has not been downloaded correctly except ObjectDoesNotExist as e: print e pass