def verifyFileDownload(self,resType):
""" Verify the files of a token """
hList = list()
downloadFolder = self.d.folder
for f in FileDownload.objects.filter(tokenID=self.t):
hashName = crypto.sha256(f.fileName+crypto.HASH_SEPARATOR+f.alternateName).hexdigest()
path = os.path.join(self.filesFolder,hashName+"_"+f.alternateName)
isFile = os.path.isfile(path)
if isFile:
#first compute the hash
h = crypto.sha256File(path)
#now verify the hash
sourceSignature = f.fileHash
verification = crypto.verifyRSAsignatureSHA256(h,sourceSignature,settings.PUB_KEY)
#history only if specified
if resType == constConfig.VERIFY_CHOICE_FILESHISTORY:
historyVerification = self.verifyHistory(f)
else:
historyVerification = None
hList.append({'fID': f.id,'fName':f.fileName,'fSig':crypto.sha256(f.fileHash).hexdigest(),'verificationResult':verification,'history':historyVerification})
elif not isFile:
hList.append({'fID': f.id,'fName':f.fileName,'fSig':"File does not exists on disk",'verificationResult':-1,'history': list()})
return hList
def verifyHistory(self,fileDownload):
""" Verify file history """
hList = list()
for fh in FileHistory.objects.filter(fileDownloadID=fileDownload):
#verification of revision metadata
revMeta = fh.revisionMetadata
revDownTime = getTimestamp(fh.downloadTime)
h = crypto.sha256(revMeta+crypto.HASH_SEPARATOR+revDownTime)
verification = crypto.verifyRSAsignatureSHA256(h,fh.revisionMetadataHash,settings.PUB_KEY)
hashName = crypto.sha256(fileDownload.fileName+crypto.HASH_SEPARATOR+fh.revision).hexdigest()
path = os.path.join(self.historyFolder,fileDownload.alternateName,hashName+"_"+fh.revision)
isPath = os.path.isfile(path)
if isPath:
#verification of file history
fHash = crypto.sha256File(path)
verificationFile = crypto.verifyRSAsignatureSHA256(fHash,fh.fileRevisionHash,settings.PUB_KEY)
hList.append({'hID': fh.id,'revID':fh.revision,'metadataVerificationResult': verification,'fileVerificationResult':verificationFile})
else:
hList.append({'hID': fh.id,'revID':fh.revision,'metadataVerificationResult': verification,'fileVerificationResult':"File does not exists on disk"})
return hList
def displaySingle(self, title, altName, downAltName, fileFolder):
""" Display a single file """
if fileFolder == "file" or altName == downAltName:
f = constConfig.DOWNLOAD_FILES_FOLDER
elif fileFolder == "history":
f = os.path.join(constConfig.DOWNLOAD_HISTORY_FOLDER, downAltName)
name = crypto.sha256(title + crypto.HASH_SEPARATOR +
altName).hexdigest() + "_" + altName
downloadFolder = Download.objects.get(
tokenID=self.t, threadStatus=constConfig.THREAD_TS).folder
fullPath = os.path.join(settings.DOWNLOAD_DIR, downloadFolder, f, name)
print os.path.isfile(fullPath)
print fullPath
mime = magic.Magic(mime=True)
mimeType = mime.from_file(fullPath)
if mimeType in constConfig.ALLOWED_MIME_TYPE[1:-1]:
t = Thubmnailer()
t.cacheImg(fullPath,
os.path.join(settings.DIFF_DIR, name + ".thumbnail"))
return {'name': name, 'mime': mimeType}
elif mimeType == constConfig.ALLOWED_MIME_TYPE[0]:
dest = os.path.join(settings.DIFF_DIR, name)
if not os.path.isfile(dest):
shutil.copy(fullPath, dest)
return {'name': name, 'mime': constConfig.ALLOWED_MIME_TYPE[0]}
def displaySingle(self,title,altName,downAltName,fileFolder):
""" Display a single file """
if fileFolder == "file" or altName == downAltName:
f = constConfig.DOWNLOAD_FILES_FOLDER
elif fileFolder == "history":
f = os.path.join(constConfig.DOWNLOAD_HISTORY_FOLDER,downAltName)
name = crypto.sha256(title+crypto.HASH_SEPARATOR+altName).hexdigest() + "_" + altName
downloadFolder = Download.objects.get(tokenID=self.t,threadStatus = constConfig.THREAD_TS).folder
fullPath = os.path.join(settings.DOWNLOAD_DIR, downloadFolder,f,name)
print os.path.isfile(fullPath)
print fullPath
mime = magic.Magic(mime=True)
mimeType = mime.from_file(fullPath)
if mimeType in constConfig.ALLOWED_MIME_TYPE[1:-1]:
t = Thubmnailer()
t.cacheImg(fullPath,os.path.join(settings.DIFF_DIR,name+".thumbnail"))
return {'name': name,'mime':mimeType}
elif mimeType == constConfig.ALLOWED_MIME_TYPE[0]:
dest = os.path.join(settings.DIFF_DIR,name)
if not os.path.isfile(dest):
shutil.copy(fullPath, dest)
return {'name': name, 'mime':constConfig.ALLOWED_MIME_TYPE[0]}
def verifyMetadata(self):
""" Verifiy the file metadata """
hList = list()
meta = FileMetadata.objects.get(tokenID=self.t)
metaFile = meta.metadata
mTime = getTimestamp(meta.metaTime)
#compute hash
h = crypto.sha256(metaFile+crypto.HASH_SEPARATOR+mTime)
#verify
verification = crypto.verifyRSAsignatureSHA256(h,meta.metadataHash,settings.PUB_KEY)
sig = crypto.rsaSignatureSHA256(metaFile+crypto.HASH_SEPARATOR+mTime,settings.PRIV_KEY)
#metadata hash
mSig = crypto.sha256(meta.metadataHash).hexdigest()
return ({'metaID': meta.id,'verificationResult': verification,'mSig':mSig})
def compareFromReport(self):
""" Compare the file found in the report with the ones downloaded """
res = list()
#get the report
report = openReport(self.t.cloudItem)
if report is not None:
cloudFiles = report[2]['objects']
if cloudFiles is not None:
#compute the hash of the downloaded files:
#get the files of the token
files = FileDownload.objects.filter(tokenID=self.t)
downFolder = Download.objects.get(tokenID=self.t).folder
for f in files:
localRes = list()
fileName = crypto.sha256(f.fileName +
crypto.HASH_SEPARATOR +
f.alternateName).hexdigest()
basePath = os.path.join(settings.DOWNLOAD_DIR, downFolder,
"files")
fullPath = os.path.join(basePath,
fileName + "_" + f.alternateName)
if os.path.isfile(fullPath):
fileDigest = crypto.sha256File(fullPath).hexdigest()
for cloud in cloudFiles:
#check if the CSP of the token is in the report
if self.t.serviceType in cloud[
'cloudService'].lower():
#now check if in the report we have the same file
for cloudFile in cloud['files']:
if cloudFile['hash'] == fileDigest:
localRes.append({
'csp':
cloud['cloudService'],
'files':
cloudFile
})
res.append({'file': f, 'comparableFiles': localRes})
return res
def downloadFiles(self, simulateDownload=False):
""" Download files """
#used for test
if simulateDownload is True:
time.sleep(constConfig.TEST_THREAD_SLEEP_TIME)
return downStatus
downDirFullSub = os.path.join(self.downloadDir,
constConfig.DOWNLOAD_FILES_FOLDER)
if not os.path.isdir(downDirFullSub):
os.mkdir(downDirFullSub)
#for each folder
for c in self.metadata:
#for each file in folder
for f in c['contents']:
if not f['is_dir']: # if is a file
#compute the alternateName
altName = dropboxAlternateName(f['path'], f['modified'])
bName = os.path.basename(f['path'])
try:
with self.service.get_file(f['path']) as f:
hashName = crypto.sha256(bName +
crypto.HASH_SEPARATOR +
altName).hexdigest()
fullPath = os.path.join(downDirFullSub,
hashName + "_" + altName)
outF = open(fullPath, "wb+")
outF.write(f.read())
outF.close()
h = crypto.rsaSignatureSHA256(
fullPath, settings.PRIV_KEY, True)
fDb = FileDownload(fileName=bName,
alternateName=altName,
status=1,
tokenID=self.t,
fileHash=h)
fDb.save()
except dropbox.rest.ErrorResponse as e:
f = FileDownload(fileName=bName,
alternateName=altName,
status=e.status,
tokenID=self.t,
fileHash="-")
f.save()
def downloadFiles(self,simulateDownload = False): """ Download file with google drive """ #used for tests if simulateDownload is True: #simulate the download by waiting 10 seconds time.sleep(constConfig.TEST_THREAD_SLEEP_TIME) return downStatus #get download folder downDirFullSub = os.path.join(self.downloadDir,constConfig.DOWNLOAD_FILES_FOLDER) if not os.path.isdir(downDirFullSub): os.mkdir(downDirFullSub) #iterate over file and write to disk for item in self.metadata: if 'downloadUrl' in item: url = item['downloadUrl'] elif 'exportLinks' in item: url = item['exportLinks']["application/pdf"] else: url = None if url != None: fileDb = None try: resp, content = self.service._http.request(url) if resp.status == 200: hashFileName = crypto.sha256(item['title']+crypto.HASH_SEPARATOR+item['id']) fullName = os.path.join(downDirFullSub,hashFileName.hexdigest()+ "_" + item['id']) with open(fullName,"wb+") as f: f.write(content) #compute hash h = crypto.rsaSignatureSHA256(fullName,settings.PRIV_KEY,True) else: h = "-" fileDb = FileDownload(fileName=item['title'],alternateName=item['id'],status=resp.status,tokenID=self.t,fileHash=h) fileDb.save() except errors.HttpError, e: #store this entry with the exception code fileDb = FileDownload(fileName=item['item'],alternateName=item['id'],status=e.resp.status,tokenID=self.t,fileHash="-") fileDb.save()
def createZIPtoVerify(self):
""" Verify a ZIP by computing its signature """
#the download has completed and there is not another ZIP verified
if self.download.threadStatus == constConfig.THREAD_DOWN_FH and self.download.verificationZIP == False:
dstPath = self.createZIP()
#compute the rsa signature
signature = crypto.rsaSignatureSHA256(dstPath,settings.PRIV_KEY,True)
self.download.verificationZIP = True
self.download.verificationZIPSignatureHash = crypto.sha256(signature).hexdigest()
self.download.verificationZIPSignature = signature
self.download.save()
return True
else:
raise Exception("Download not complete or ZIP already exists.")
def createZIPtoVerify(self):
""" Verify a ZIP by computing its signature """
#the download has completed and there is not another ZIP verified
if self.download.threadStatus == constConfig.THREAD_DOWN_FH and self.download.verificationZIP == False:
dstPath = self.createZIP()
#compute the rsa signature
signature = crypto.rsaSignatureSHA256(dstPath, settings.PRIV_KEY,
True)
self.download.verificationZIP = True
self.download.verificationZIPSignatureHash = crypto.sha256(
signature).hexdigest()
self.download.verificationZIPSignature = signature
self.download.save()
return True
else:
raise Exception("Download not complete or ZIP already exists.")
def compareFromReport(self):
""" Compare the file found in the report with the ones downloaded """
res = list()
#get the report
report = openReport(self.t.cloudItem)
if report is not None:
cloudFiles = report[2]['objects']
if cloudFiles is not None:
#compute the hash of the downloaded files:
#get the files of the token
files = FileDownload.objects.filter(tokenID=self.t)
downFolder = Download.objects.get(tokenID=self.t).folder
for f in files:
localRes = list()
fileName = crypto.sha256(f.fileName+crypto.HASH_SEPARATOR+f.alternateName).hexdigest()
basePath = os.path.join(settings.DOWNLOAD_DIR,downFolder,"files")
fullPath = os.path.join(basePath,fileName+"_"+f.alternateName)
if os.path.isfile(fullPath):
fileDigest = crypto.sha256File(fullPath).hexdigest()
for cloud in cloudFiles:
#check if the CSP of the token is in the report
if self.t.serviceType in cloud['cloudService'].lower():
#now check if in the report we have the same file
for cloudFile in cloud['files']:
if cloudFile['hash'] == fileDigest:
localRes.append({'csp':cloud['cloudService'],'files':cloudFile})
res.append({'file':f,'comparableFiles':localRes})
return res
def __init__(self,cloudItem,browserParam):
self.fb = None
#check that an import exists
imp = Upload.objects.get(cloudItemID=cloudItem)
#compute import path
importPath = crypto.sha256(imp.fileName+crypto.HASH_SEPARATOR+getTimestamp(imp.uploadDate)).hexdigest()
self.filePath = os.path.join(settings.UPLOAD_DIR,str(cloudItem.id),importPath,imp.fileName)
profile = str(browserParam.split("_",1)[1])
#windows profile are stored with Profile\in front
if profile.startswith("Profile"):
profile = profile[9:]
if int(browserParam[0]) == constConfig.HISTORY_FORM_CHROME:
self.fb = GoogleChromeFiles(self.filePath,profile)
elif int(browserParam[0]) == constConfig.HISTORY_FORM_FF:
self.fb = FirefoxFiles(self.filePath,profile)
def downloadFiles(self,simulateDownload = False): """ Download files """ #used for test if simulateDownload is True: time.sleep(constConfig.TEST_THREAD_SLEEP_TIME) return downStatus downDirFullSub = os.path.join(self.downloadDir, constConfig.DOWNLOAD_FILES_FOLDER) if not os.path.isdir(downDirFullSub): os.mkdir(downDirFullSub) #for each folder for c in self.metadata: #for each file in folder for f in c['contents']: if not f['is_dir']: # if is a file #compute the alternateName altName = dropboxAlternateName(f['path'],f['modified']) bName = os.path.basename(f['path']) try: with self.service.get_file(f['path']) as f: hashName = crypto.sha256(bName+crypto.HASH_SEPARATOR+altName).hexdigest() fullPath = os.path.join(downDirFullSub,hashName+"_"+altName) outF = open(fullPath,"wb+") outF.write(f.read()) outF.close() h = crypto.rsaSignatureSHA256(fullPath,settings.PRIV_KEY,True) fDb = FileDownload(fileName=bName,alternateName=altName,status=1,tokenID=self.t,fileHash=h) fDb.save() except dropbox.rest.ErrorResponse as e: f = FileDownload(fileName=bName,alternateName=altName,status=e.status,tokenID=self.t,fileHash="-") f.save()
def downloadHistory(self,simulateDownload = False): """ Download the history for a file """ #used for tests if simulateDownload is True: #simulate the download by waiting 10 seconds time.sleep(constConfig.TEST_THREAD_SLEEP_TIME) return downStatus downDirHistory = os.path.join(self.downloadDir,constConfig.DOWNLOAD_HISTORY_FOLDER) if not os.path.isdir(downDirHistory): os.mkdir(downDirHistory) for item in self.metadata: #folders do not support revision if item['mimeType'] != 'application/vnd.google-apps.folder': fh = None try: fileDownload = FileDownload.objects.get(fileName=item['title'],alternateName=item['id'],tokenID=self.t,status=200) try: #get revisions for this file revs = self.service.revisions().list(fileId=item['id']).execute() if len(revs['items']) > 1: #create a folder for this file revPath = os.path.join(downDirHistory,item['id']) if not os.path.isdir(revPath): os.mkdir(revPath) for r in revs['items']: if 'exportLinks' in r: url = r['exportLinks']["application/pdf"] elif 'downloadUrl' in r: url = r['downloadUrl'] else: url = None if url != None: resp, content = self.service._http.request(url) revItem = base64.b64encode(json.dumps(r)) revID = r['id'] hashFileName = crypto.sha256(item['title']+crypto.HASH_SEPARATOR+revID) downloadTime = timezone.now() revisionMetadataHash = crypto.rsaSignatureSHA256( revItem+crypto.HASH_SEPARATOR+getTimestamp(downloadTime), settings.PRIV_KEY) #if the response is affirmative store the file if resp.status == 200: fullName = os.path.join(revPath,hashFileName.hexdigest()+"_"+revID) with open(fullName,"wb+") as f: f.write(content) # compute hash fileRevisionHash = crypto.rsaSignatureSHA256(fullName,settings.PRIV_KEY,True) else: fileRevisionHash = "-" fh = FileHistory( revision=revID, status=resp.status, fileDownloadID=fileDownload, revisionMetadata=revItem, downloadTime=downloadTime, fileRevisionHash=fileRevisionHash, revisionMetadataHash=revisionMetadataHash ) fh.save() # HTTP when requesting history list or downloading file except errors.HttpError, e: fh = FileHistory(revision="-",status=e.resp.status,fileDownloadID=fileDownload,revisionMetadata="-",fileRevisionHash="-",revisionMetadataHash="-") fh.save() #we cannot download the history for a file that has not been downloaded correctly except ObjectDoesNotExist as e: print e pass
def searchMetaData(request,form,tokenID,cloudItem,start):
""" Make a search through the metadata """
dajax = Dajax()
try:
t = parseAjaxParam(tokenID)
ciChk = checkCloudItem(cloudItem,request.user.id)
tknObj = checkAccessToken(t,ciChk)
searchStep = 100
f = MetaSearch(deserialize_form(form))
if f.is_valid():
startResTime = time.time()
#compute hash of the search form for the cache
searchHash = crypto.sha256(form).hexdigest()
"""searchHash = crypto.sha256(f.cleaned_data['formType'][0]+crypto.HASH_SEPARATOR+
f.cleaned_data['email']+crypto.HASH_SEPARATOR+
f.cleaned_data['filename']+crypto.HASH_SEPARATOR+
f.cleaned_data['givenname']+crypto.HASH_SEPARATOR+
f.cleaned_data['resType'][0]+crypto.HASH_SEPARATOR+
f.cleaned_data['mimeType']+crypto.HASH_SEPARATOR+
str(f.cleaned_data['startDate'])+crypto.HASH_SEPARATOR+
str(f.cleaned_data['endDate'])
).hexdigest()"""
if "searchCache" in request.session and request.session['searchCacheID'] == searchHash:
res = json.loads(request.session["searchCache"])
else:
mc = MetadataController(tknObj)
res = mc.metadataSearch(
int(f.cleaned_data['formType'][0]),
f.cleaned_data['email'],
f.cleaned_data['filename'],
f.cleaned_data['givenname'],
int(f.cleaned_data['resType'][0]),
int(f.cleaned_data['mimeType']),
f.cleaned_data['startDate'],
f.cleaned_data['endDate']
)
request.session["searchCacheID"] = searchHash
request.session["searchCache"] = json.dumps(res)
#computation for pager
totalPages = int(math.ceil(float(len(res))/100.0))
resultsSlice = res[start:(start+searchStep)]
stopResTime = time.time()
parsedTable = render_to_string("dashboard/cloudservice/searchTable.html", {'data': resultsSlice,'totalPages':range(totalPages),'totalRes':len(res),'resTime': stopResTime-startResTime,'platform':tknObj.serviceType})
dajax.assign("#searchRes","innerHTML",parsedTable)
dajax.assign("#searchError","innerHTML","")
dajax.remove_css_class("#searchError",['alert','alert-danger'])
else:
dajax.assign("#searchError","innerHTML","Please fill all fields")
dajax.add_css_class("#searchError",['alert','alert-danger'])
except Exception as e:
dajax.assign("#searchError","innerHTML",formatException(e))
dajax.add_css_class("#searchError",['alert','alert-danger'])
return dajax.json()
def fileInfo(self):
""" Displays information about the downloaded files """
history = []
rowElem = list()
self.story.append(Paragraph("Downloaded Files",styles['Heading1']))
#files
files = DbInterface.getAllFileDownload(self.t)
data = [[Paragraph("<b>ID</b>",self.normalStyle),
Paragraph("<b>File Name</b>",self.normalStyle),
Paragraph("<b>Download Time</b>",self.normalStyle),
Paragraph("<b>Signature</b>",self.normalStyle)]]
for f in files:
line = [Paragraph(str(f.id),self.normalStyle),
Paragraph(f.fileName,self.wrappedStyle),
Paragraph(str(timezone.localtime(f.downloadTime)),self.wrappedStyle),
Paragraph(crypto.sha256(f.fileHash).hexdigest(),self.wrappedStyle)
]
data.append(line)
hist = DbInterface.getHistoryForFile(f)
#if we have an history
if len(hist) > 0:
#history.append([f.id,DbInterface.getHistoryForFile(f)])
histData = [[Paragraph("<b>ID</b>",self.normalStyle),
Paragraph("<b>Revision</b>",self.normalStyle),
Paragraph("<b>Download Time</b>",self.normalStyle),
Paragraph("<b>Signature</b>",self.normalStyle)
]]
for h in hist:
histData.append([
Paragraph(str(h.id),self.normalStyle),
Paragraph(h.revision,self.normalStyle),
Paragraph(str(timezone.localtime(h.downloadTime)),self.normalStyle),
Paragraph(crypto.sha256(h.fileRevisionHash).hexdigest(),self.normalStyle)
])
histT = Table(histData, colWidths=(1.5*cm,6*cm,6*cm,6*cm))
histT.setStyle(TableStyle([('INNERGRID', (0,0), (-1,-1), 0.25, colors.black),
('BOX', (0,0), (-1,-1), 0.25, colors.black),]))
data.append(["",histT,"",""])
#row to which apply the colspan
rowElem.append(('BACKGROUND',(0,len(data)-1),(0,len(data)-1),colors.green))
rowElem.append(('SPAN',(1,len(data)-1),(3,len(data)-1)))
t = Table(data, colWidths=(2*cm,9*cm,5*cm,8.5*cm))
styleParam = [('INNERGRID', (0,0), (-1,-1), 0.25, colors.black),('BOX', (0,0), (-1,-1), 0.25, colors.black),]
styleParam.extend(rowElem)
print rowElem
print styleParam
ts = TableStyle(styleParam)
t.setStyle(ts)
self.story.append(t)
self.story.append(PageBreak())
def compareTwo(self, revOneID, revTwoID, altName):
""" Compare two revision of the same file and check for diff """
finalDiffName = None
downloadFolder = Download.objects.get(
tokenID=self.t, threadStatus=constConfig.THREAD_TS).folder
diffFile = FileDownload.objects.get(tokenID=self.t,
alternateName=altName)
#diff file full path
diffPath = os.path.join(settings.DOWNLOAD_DIR, downloadFolder,
constConfig.DOWNLOAD_HISTORY_FOLDER,
diffFile.alternateName)
#get the two file
nameOne = crypto.sha256(diffFile.fileName + crypto.HASH_SEPARATOR +
revOneID).hexdigest()
nameTwo = crypto.sha256(diffFile.fileName + crypto.HASH_SEPARATOR +
revTwoID).hexdigest()
revOnePath = os.path.join(diffPath, nameOne + "_" + revOneID)
revTwoPath = os.path.join(diffPath, nameTwo + "_" + revTwoID)
#check if dropbox that one of the file is not the original
if self.t.serviceType == constConfig.CSP_DROPBOX:
#try to get the revision from the filedownload table
if diffFile.alternateName == revOneID:
assumeNameOne = crypto.sha256(
diffFile.fileName + crypto.HASH_SEPARATOR +
diffFile.alternateName).hexdigest()
assumedPath = os.path.join(
settings.DOWNLOAD_DIR, downloadFolder,
constConfig.DOWNLOAD_FILES_FOLDER,
assumeNameOne + "_" + diffFile.alternateName)
#overwrite revOnePath only if the path exists in the files folder, otherwise is in the deleted folder
if os.path.isfile(assumedPath):
revOnePath = assumedPath
elif diffFile.alternateName == revTwoID:
assumeNameTwo = crypto.sha256(
diffFile.fileName + crypto.HASH_SEPARATOR +
diffFile.alternateName).hexdigest()
assumedPath = os.path.join(
settings.DOWNLOAD_DIR, downloadFolder,
constConfig.DOWNLOAD_FILES_FOLDER,
assumeNameTwo + "_" + diffFile.alternateName)
if os.path.isfile(assumedPath):
revTwoPath = assumedPath
#check that the two path actually exists. This because the actual file in dropbox (rightmost in the file history time line) does not exists if it has been deleted. So we will have an entry in the timeline for a version of a file that does not exist.
if not os.path.isfile(revOnePath) or not os.path.isfile(revTwoPath):
raise Exception(
"One of the two file does not exist. (Is this a deleted file on Dropbox?)"
)
print revOnePath
print revTwoPath
#check allowed mime
mime = magic.Magic(mime=True)
mimeOne = mime.from_file(revOnePath)
mimeTwo = mime.from_file(revTwoPath)
if mimeOne not in constConfig.ALLOWED_MIME_TYPE or mimeTwo not in constConfig.ALLOWED_MIME_TYPE:
raise Exception("File type not supported")
data = None
mimeList = list()
#pdf files
if mimeOne == mimeTwo == constConfig.ALLOWED_MIME_TYPE[0]:
resultDiffName = "diff_" + str(
self.t.id) + "_" + revOneID + "_" + revTwoID + ".pdf"
diffName = self.pdfDiff(revOnePath, revTwoPath, diffPath,
resultDiffName)
data = {"diffName": diffName}
mimeList.append(constConfig.ALLOWED_MIME_TYPE[0])
#images
elif mimeOne and mimeTwo in constConfig.ALLOWED_MIME_TYPE[1:-1]:
hash1, hash2 = self.imgDiff(revOnePath, revTwoPath)
mimeList.append(mime.from_file(revOnePath))
mimeList.append(mime.from_file(revTwoPath))
#build data dictionary
data = {
'hash1': hash1,
'hash2': hash2,
'file1': diffFile.fileName + "_" + revOneID,
'file2': diffFile.fileName + "_" + revTwoID
}
return {"filename": diffFile.fileName, "mime": mimeList, "data": data}
def compareTwo(self,revOneID,revTwoID,altName):
""" Compare two revision of the same file and check for diff """
finalDiffName = None
downloadFolder = Download.objects.get(tokenID=self.t,threadStatus = constConfig.THREAD_TS).folder
diffFile = FileDownload.objects.get(tokenID=self.t,alternateName=altName)
#diff file full path
diffPath = os.path.join(settings.DOWNLOAD_DIR,downloadFolder,constConfig.DOWNLOAD_HISTORY_FOLDER,diffFile.alternateName)
#get the two file
nameOne = crypto.sha256(diffFile.fileName+crypto.HASH_SEPARATOR+revOneID).hexdigest()
nameTwo = crypto.sha256(diffFile.fileName+crypto.HASH_SEPARATOR+revTwoID).hexdigest()
revOnePath = os.path.join(diffPath,nameOne+"_"+revOneID)
revTwoPath = os.path.join(diffPath,nameTwo+"_"+revTwoID)
#check if dropbox that one of the file is not the original
if self.t.serviceType == constConfig.CSP_DROPBOX:
#try to get the revision from the filedownload table
if diffFile.alternateName == revOneID:
assumeNameOne = crypto.sha256(diffFile.fileName+crypto.HASH_SEPARATOR+diffFile.alternateName).hexdigest()
assumedPath = os.path.join(settings.DOWNLOAD_DIR,downloadFolder,constConfig.DOWNLOAD_FILES_FOLDER,assumeNameOne+"_"+diffFile.alternateName)
#overwrite revOnePath only if the path exists in the files folder, otherwise is in the deleted folder
if os.path.isfile(assumedPath):
revOnePath = assumedPath
elif diffFile.alternateName == revTwoID:
assumeNameTwo = crypto.sha256(diffFile.fileName+crypto.HASH_SEPARATOR+diffFile.alternateName).hexdigest()
assumedPath = os.path.join(settings.DOWNLOAD_DIR,downloadFolder,constConfig.DOWNLOAD_FILES_FOLDER,assumeNameTwo+"_"+diffFile.alternateName)
if os.path.isfile(assumedPath):
revTwoPath = assumedPath
#check that the two path actually exists. This because the actual file in dropbox (rightmost in the file history time line) does not exists if it has been deleted. So we will have an entry in the timeline for a version of a file that does not exist.
if not os.path.isfile(revOnePath) or not os.path.isfile(revTwoPath):
raise Exception("One of the two file does not exist. (Is this a deleted file on Dropbox?)")
print revOnePath
print revTwoPath
#check allowed mime
mime = magic.Magic(mime=True)
mimeOne = mime.from_file(revOnePath)
mimeTwo = mime.from_file(revTwoPath)
if mimeOne not in constConfig.ALLOWED_MIME_TYPE or mimeTwo not in constConfig.ALLOWED_MIME_TYPE:
raise Exception("File type not supported")
data = None
mimeList = list()
#pdf files
if mimeOne == mimeTwo == constConfig.ALLOWED_MIME_TYPE[0]:
resultDiffName = "diff_"+str(self.t.id)+"_"+revOneID+"_"+revTwoID+".pdf"
diffName = self.pdfDiff(revOnePath,revTwoPath,diffPath,resultDiffName)
data = {"diffName":diffName}
mimeList.append(constConfig.ALLOWED_MIME_TYPE[0])
#images
elif mimeOne and mimeTwo in constConfig.ALLOWED_MIME_TYPE[1:-1]:
hash1,hash2 = self.imgDiff(revOnePath,revTwoPath)
mimeList.append(mime.from_file(revOnePath))
mimeList.append(mime.from_file(revTwoPath))
#build data dictionary
data = {'hash1': hash1,'hash2': hash2, 'file1':diffFile.fileName+"_"+revOneID,'file2':diffFile.fileName+"_"+revTwoID}
return {"filename": diffFile.fileName,"mime": mimeList, "data": data}
def fileInfo(self):
""" Displays information about the downloaded files """
history = []
rowElem = list()
self.story.append(Paragraph("Downloaded Files", styles['Heading1']))
#files
files = DbInterface.getAllFileDownload(self.t)
data = [[
Paragraph("<b>ID</b>", self.normalStyle),
Paragraph("<b>File Name</b>", self.normalStyle),
Paragraph("<b>Download Time</b>", self.normalStyle),
Paragraph("<b>Signature</b>", self.normalStyle)
]]
for f in files:
line = [
Paragraph(str(f.id), self.normalStyle),
Paragraph(f.fileName, self.wrappedStyle),
Paragraph(str(timezone.localtime(f.downloadTime)),
self.wrappedStyle),
Paragraph(
crypto.sha256(f.fileHash).hexdigest(), self.wrappedStyle)
]
data.append(line)
hist = DbInterface.getHistoryForFile(f)
#if we have an history
if len(hist) > 0:
#history.append([f.id,DbInterface.getHistoryForFile(f)])
histData = [[
Paragraph("<b>ID</b>", self.normalStyle),
Paragraph("<b>Revision</b>", self.normalStyle),
Paragraph("<b>Download Time</b>", self.normalStyle),
Paragraph("<b>Signature</b>", self.normalStyle)
]]
for h in hist:
histData.append([
Paragraph(str(h.id), self.normalStyle),
Paragraph(h.revision, self.normalStyle),
Paragraph(str(timezone.localtime(h.downloadTime)),
self.normalStyle),
Paragraph(
crypto.sha256(h.fileRevisionHash).hexdigest(),
self.normalStyle)
])
histT = Table(histData,
colWidths=(1.5 * cm, 6 * cm, 6 * cm, 6 * cm))
histT.setStyle(
TableStyle([
('INNERGRID', (0, 0), (-1, -1), 0.25, colors.black),
('BOX', (0, 0), (-1, -1), 0.25, colors.black),
]))
data.append(["", histT, "", ""])
#row to which apply the colspan
rowElem.append(('BACKGROUND', (0, len(data) - 1),
(0, len(data) - 1), colors.green))
rowElem.append(
('SPAN', (1, len(data) - 1), (3, len(data) - 1)))
t = Table(data, colWidths=(2 * cm, 9 * cm, 5 * cm, 8.5 * cm))
styleParam = [
('INNERGRID', (0, 0), (-1, -1), 0.25, colors.black),
('BOX', (0, 0), (-1, -1), 0.25, colors.black),
]
styleParam.extend(rowElem)
print rowElem
print styleParam
ts = TableStyle(styleParam)
t.setStyle(ts)
self.story.append(t)
self.story.append(PageBreak())
def downloadHistory(self, simulateDownload=False):
""" Download the history for dropbox """
#used for test
if simulateDownload is True:
time.sleep(constConfig.TEST_THREAD_SLEEP_TIME)
return downStatus
downDirFullSub = os.path.join(self.downloadDir,
constConfig.DOWNLOAD_HISTORY_FOLDER)
if not os.path.isdir(downDirFullSub):
os.mkdir(downDirFullSub)
for c in self.metadata:
#for each file in folder
for f in c['contents']:
if not f['is_dir']:
rev = self.service.revisions(f['path'])
if len(rev) > 1: # one revision means original file
#compute alternate name for db lookup, from the first revision that is the original file
modified = rev[0]['modified']
path = rev[0]['path']
s = path.encode('utf-8') + modified.encode('utf-8')
altName = md5.new(s).hexdigest()
bName = os.path.basename(path)
#get file download id
try:
fDown = FileDownload.objects.get(
fileName=bName,
alternateName=altName,
tokenID=self.t,
status=1)
del rev[0]
# create a directory to store file revision
revPath = os.path.join(downDirFullSub, altName)
if not os.path.isdir(revPath):
os.mkdir(revPath)
for r in rev:
revID = r['rev']
hashName = crypto.sha256(
bName + crypto.HASH_SEPARATOR +
revID).hexdigest()
fullPath = os.path.join(
revPath, hashName + "_" + revID)
try:
#get revision
with self.service.get_file(
f['path'], revID) as revF:
outF = open(fullPath, "wb+")
outF.write(revF.read())
outF.close()
#hash
rEnc = base64.b64encode(json.dumps(r))
downloadTime = timezone.now()
fileRevisionHash = crypto.rsaSignatureSHA256(
fullPath, settings.PRIV_KEY, True)
revisionMetadataHash = crypto.rsaSignatureSHA256(
rEnc + crypto.HASH_SEPARATOR +
format(downloadTime, "U"),
settings.PRIV_KEY)
fDb = FileHistory(
revision=revID,
status=1,
fileDownloadID=fDown,
revisionMetadata=rEnc,
downloadTime=downloadTime,
revisionMetadataHash=
revisionMetadataHash,
fileRevisionHash=fileRevisionHash)
fDb.save()
#when there is an error with the download of the history
except dropbox.rest.ErrorResponse as e:
fDb = FileHistory(revision=revID,
status=e.status,
fileDownloadID=fDown,
revisionMetadata="-",
fileRevisionHash="-",
revisionMetadataHash="-")
fDb.save()
#we cannot download the history of a file that has not been downloaded correctly
except ObjectDoesNotExist as e:
print e
pass
def downloadHistory(self,simulateDownload = False):
""" Download the history for dropbox """
#used for test
if simulateDownload is True:
time.sleep(constConfig.TEST_THREAD_SLEEP_TIME)
return downStatus
downDirFullSub = os.path.join(self.downloadDir,constConfig.DOWNLOAD_HISTORY_FOLDER)
if not os.path.isdir(downDirFullSub):
os.mkdir(downDirFullSub)
for c in self.metadata:
#for each file in folder
for f in c['contents']:
if not f['is_dir']:
rev = self.service.revisions(f['path'])
if len(rev) > 1: # one revision means original file
#compute alternate name for db lookup, from the first revision that is the original file
modified = rev[0]['modified']
path = rev[0]['path']
s = path.encode('utf-8') + modified.encode('utf-8')
altName = md5.new(s).hexdigest()
bName = os.path.basename(path)
#get file download id
try:
fDown = FileDownload.objects.get(fileName=bName,alternateName=altName,tokenID=self.t,status=1)
del rev[0]
# create a directory to store file revision
revPath = os.path.join(downDirFullSub,altName)
if not os.path.isdir(revPath):
os.mkdir(revPath)
for r in rev:
revID = r['rev']
hashName = crypto.sha256(bName+crypto.HASH_SEPARATOR+revID).hexdigest()
fullPath = os.path.join(revPath,hashName+"_"+revID)
try:
#get revision
with self.service.get_file(f['path'],revID) as revF:
outF = open(fullPath,"wb+")
outF.write(revF.read())
outF.close()
#hash
rEnc = base64.b64encode(json.dumps(r))
downloadTime = timezone.now()
fileRevisionHash = crypto.rsaSignatureSHA256(fullPath,settings.PRIV_KEY,True)
revisionMetadataHash = crypto.rsaSignatureSHA256(rEnc+crypto.HASH_SEPARATOR+format(downloadTime,"U"),settings.PRIV_KEY)
fDb = FileHistory(
revision=revID,
status=1,
fileDownloadID=fDown,
revisionMetadata=rEnc,
downloadTime=downloadTime,
revisionMetadataHash=revisionMetadataHash,
fileRevisionHash=fileRevisionHash
)
fDb.save()
#when there is an error with the download of the history
except dropbox.rest.ErrorResponse as e:
fDb = FileHistory(revision=revID,status=e.status,fileDownloadID=fDown,revisionMetadata="-",fileRevisionHash="-",revisionMetadataHash="-")
fDb.save()
#we cannot download the history of a file that has not been downloaded correctly
except ObjectDoesNotExist as e:
print e
pass
