def Parse(self, FileName, FileData):
resp = FileParserResponse()
try:
meta = self.Metadata()
if FileName and FileName != '':
meta.set(self.Metadata.RESOURCE_NAME_KEY, FileName)
contentHandler = self.BodyContentHandler(-1)
inputStream = self.ByteArrayInputStream(FileData)
self.parser.parse(inputStream, contentHandler, meta)
try:
resp.text = contentHandler.toString()
except Exception as convEx:
resp.text = BinaryStringParser.Parse(convEx.object)
for name in meta.names():
try:
resp.meta[name] = meta.get(name)
except:
resp.meta[name] = ''
inputStream = None
contentHandler = None
if 'Content-Type' in resp.meta and ContentTypeAnalyzer.IsImageByContentType(
resp.meta['Content-Type']):
self.logger.LogMessage(
'info', 'performing ocr on {0}'.format(FileName))
ocrResp = self.ocrProxy.PerformOCR(FileData)
if ocrResp.success:
resp.text = self.NormalizeText('{0}{1}'.format(
resp.text, ocrResp.text))
resp.ocrPerformed = True
if not ocrResp.success:
self.logger.LogMessage(
'info', 'could not perform ocr on {0} {1}'.format(
FileName, ocrResp.message))
resp.thumbnail = self.GenerateThumbnail(FileData)
resp.success = True
except Exception as ex:
resp.success = False
resp.message = str(ex)
return resp
def Parse(self, FileName, FileData):
if ContentTypeAnalyzer.IsPdf(FileName):
return self.pdfParser.Parse(FileName, FileData)
return self.tikaParser.Parse(FileName, FileData)
def SetImageTag(self, AmbarFile):
if ContentTypeAnalyzer.IsImageByContentType(
AmbarFile['content']['type']):
self.AddTagToAmbarFile(AmbarFile['file_id'],
AmbarFile['meta']['full_name'],
self.AUTO_TAG_TYPE, 'image')
def SetArchiveTag(self, AmbarFile):
if ContentTypeAnalyzer.IsArchive(AmbarFile['meta']['full_name']):
self.AddTagToAmbarFile(AmbarFile['file_id'],
AmbarFile['meta']['full_name'],
self.AUTO_TAG_TYPE, 'archive')
def SetVideoTag(self, AmbarFile):
if ContentTypeAnalyzer.IsVideoByContentType(
AmbarFile['content']['type']):
self.AddTagToAmbarFile(AmbarFile['file_id'],
AmbarFile['meta']['full_name'],
self.AUTO_TAG_TYPE, 'video')
def SetTextTag(self, AmbarFile):
if ContentTypeAnalyzer.IsTextByContentType(
AmbarFile['content']['type']):
self.AddTagToAmbarFile(AmbarFile['file_id'],
AmbarFile['meta']['full_name'],
self.AUTO_TAG_TYPE, 'text')
def ProcessFile(message):
try:
meta = message['meta']
event = message['event']
sha = None
logger.LogMessage('verbose', '{0} task received for {1}'.format(event, meta['full_name']))
if ('sha' in message):
sha = message['sha']
fileId = sha256('{0}{1}'.format(meta['source_id'],meta['full_name']).encode('utf-8')).hexdigest()
if (event == 'unlink'):
apiResp = apiProxy.HideFile(fileId)
if not apiResp.Success:
logger.LogMessage('error', 'error hidding file for {0} {1}'.format(meta['full_name'], apiResp.message))
return False
if apiResp.Ok:
logger.LogMessage('verbose', 'removed {0}'.format(meta['full_name']))
return True
if not apiResp.NotFound:
logger.LogMessage('error', 'error hidding file {0} {1} code: {2}'.format(meta['full_name'], apiResp.message, apiResp.code))
return False
return True
if (event != 'add' and event != 'change'):
print('Ignoring {0}'.format(event))
return True
apiResp = apiProxy.CheckIfMetaExists(meta)
if not apiResp.Success:
logger.LogMessage('error', 'error checking meta existance for {0} {1}'.format(meta['full_name'], apiResp.message))
return False
if apiResp.Ok:
logger.LogMessage('verbose', 'meta found for {0}'.format(meta['full_name']))
return True
if not apiResp.NotFound:
logger.LogMessage('error', 'error checking meta existance for {0} {1} {2}'.format(meta['full_name'], apiResp.code, apiResp.message))
return False
apiResp = apiProxy.UnhideFile(fileId)
if not apiResp.Success:
logger.LogMessage('error', 'error unhiding file {0} {1}'.format(meta['full_name'], apiResp.message))
return False
if not (apiResp.Ok or apiResp.NotFound):
logger.LogMessage('error', 'error unhiding file, unexpected response code {0} {1} {2}'.format(fileMeta.full_name, apiResp.code, apiResp.message))
return False
fileMeta = AmbarFileMeta.Init(meta)
if not fileMeta.initialized:
logger.LogMessage('error', 'error initializing file meta {0}'.format(fileMeta.message))
return False
if (sha):
apiResp = apiProxy.DownloadFileBySha(sha)
else:
apiResp = apiProxy.DownloadFile(fileMeta.full_name)
if not apiResp.Success:
logger.LogMessage('error', 'error downloading file {0} {1}'.format(fileMeta.full_name, apiResp.message))
return False
if not apiResp.Ok:
logger.LogMessage('error', 'error downloading file {0} {1} code: {2}'.format(fileMeta.full_name, apiResp.message, apiResp.code))
return False
fileData = apiResp.payload
sha = sha256(fileData).hexdigest()
hasParsedContent = False
fileContent = {}
apiResp = apiProxy.GetParsedFileContentFields(sha)
if not apiResp.Success:
logger.LogMessage('error', 'error retrieving parsed file content fields {0} {1}'.format(
fileMeta.full_name, apiResp.message))
return False
if not (apiResp.Ok or apiResp.NotFound):
logger.LogMessage('error', 'error retrieving parsed file content fields {0} {1} {2}'.format(
fileMeta.full_name, apiResp.code, apiResp.message))
return False
if apiResp.Ok:
hasParsedContent = True
fileContent = apiResp.payload
if hasParsedContent:
apiResp = apiProxy.GetParsedFileContent(sha)
if not apiResp.Success:
logger.LogMessage('error', 'error retrieving parsed file content {0} {1}'.format(
fileMeta.full_name, apiResp.message))
return False
if not (apiResp.Ok or apiResp.NotFound):
logger.LogMessage('error', 'error retrieving parsed file content {0} {1} {2}'.format(
fileMeta.full_name, apiResp.code, apiResp.message))
return False
if apiResp.NotFound:
hasParsedContent = False
if apiResp.Ok:
hasParsedContent = True
fileContent['text'] = apiResp.payload.decode('utf-8', 'ignore')
logger.LogMessage(
'verbose', 'parsed content found {0}'.format(fileMeta.full_name))
if not hasParsedContent:
# checking if file is archive
if ContentTypeAnalyzer.IsArchive(fileMeta.short_name):
archiveProcessor.Process(fileData, fileMeta, fileMeta.source_id)
# checking if file is pst
if ContentTypeAnalyzer.IsPst(fileMeta.short_name):
pstProcessor.Process(fileData, fileMeta, fileMeta.source_id)
# extracting
logger.LogMessage('verbose', 'parsing {0}'.format(fileMeta.full_name))
fileParserResp = fileParser.Parse(fileMeta.short_name, fileData)
if not fileParserResp.success:
logger.LogMessage('error', 'error parsing {0} {1}'.format(
fileMeta.full_name, fileParserResp.message))
return False
logger.LogMessage(
'verbose', 'successfully parsed {0}'.format(fileMeta.full_name))
# building Ambar File Content
fileContent = AmbarFileContent.Init(fileParserResp, sys.getsizeof(fileData))
# submitting thumbnail
if fileParserResp.thumbnail:
logger.LogMessage(
'verbose', 'submitting thumbnail {0}'.format(fileMeta.full_name))
apiResp = apiProxy.SubmitThumbnail(
sha, fileParserResp.thumbnail[0])
if not apiResp.Success:
logger.LogMessage('error', 'error submitting thumbnail to Api {0} {1}'.format(
fileMeta.full_name, apiResp.message))
return False
if not apiResp.Ok:
logger.LogMessage('error', 'error submitting thumbnail to Api, unexpected response code {0} {1} {2}'.format(
fileMeta.full_name, apiResp.code, apiResp.message))
return False
fileContent.thumb_available = True
logger.LogMessage('verbose', 'thumbnail submited {0}'.format(fileMeta.full_name))
# submitting parsed text to Api
if not ContentTypeAnalyzer.IsArchive(fileMeta.short_name) and not ContentTypeAnalyzer.IsPst(fileMeta.short_name):
logger.LogMessage('verbose', 'submitting parsed text {0}'.format(fileMeta.full_name))
apiResp = apiProxy.SubmitExtractedContent(
sha, fileContent.text.encode(encoding='utf_8', errors='ignore'))
if not apiResp.Success:
logger.LogMessage('error', 'error submitting parsed text to Api {0} {1}'.format(fileMeta.full_name, apiResp.message))
return False
if not apiResp.Ok:
logger.LogMessage('error', 'error submitting parsed text to Api, unexpected response code {0} {1} {2}'.format(fileMeta.full_name, apiResp.code, apiResp.message))
return False
logger.LogMessage('verbose', 'parsed text submited {0}'.format(fileMeta.full_name))
# submitting processed file to Api
logger.LogMessage('verbose', 'submitting parsed content {0}'.format(fileMeta.full_name))
ambarFile = {}
ambarFile['content'] = fileContent.Dict if isinstance(fileContent, AmbarFileContent) else fileContent
ambarFile['meta'] = fileMeta.Dict
ambarFile['sha256'] = sha
ambarFile['file_id'] = fileId
ambarFile['indexed_datetime'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
apiResp = apiProxy.SubmitProcessedFile(fileId, json.dumps(dict(ambarFile)).encode(encoding='utf_8', errors='ignore'))
if not apiResp.Success:
logger.LogMessage('error', 'error submitting parsed content to Api {0} {1}'.format(
fileMeta.full_name, apiResp.message))
return False
if not (apiResp.Ok or apiResp.Created):
logger.LogMessage('error', 'error submitting parsed content to Api, unexpected response code {0} {1} {2}'.format(
fileMeta.full_name, apiResp.code, apiResp.message))
return False
logger.LogMessage('verbose', 'parsed content submited {0}'.format(fileMeta.full_name))
apiResp = apiProxy.AddMetaIdToCache(fileMeta.id)
if not apiResp.Success:
logger.LogMessage('error', 'error adding meta id to cache {0} {1}'.format(fileMeta.full_name, apiResp.message))
return False
if not apiResp.Ok:
logger.LogMessage('error', 'error adding meta id to cache, unexpected response code {0} {1} {2}'.format(fileMeta.full_name, apiResp.code, apiResp.message))
return False
# removing original file
if not preserveOriginals:
apiResp = apiProxy.RemoveFileContent(sha)
if not apiResp.Success:
logger.LogMessage('error', 'error removing original file from Ambar for {0} {1}'.format(fileMeta.full_name, apiResp.message))
return False
if not (apiResp.Ok or apiResp.NotFound):
logger.LogMessage('error', 'error removing original file from Ambar for {0}, unexpected response code {1} {2}'.format(fileMeta.full_name, apiResp.code, apiResp.message))
return False
if apiResp.Ok:
logger.LogMessage(
'verbose', 'original file removed from Ambar for {0}'.format(fileMeta.full_name))
## tags
apiResp = apiProxy.RemoveAutoTags(fileId)
if not apiResp.Success:
logger.LogMessage('error', 'error removing autotags {0} {1}'.format(fileMeta.full_name, apiResp.message))
return False
if not apiResp.Ok:
logger.LogMessage('error', 'error removing autotags, unexpected response code {0} {1} {2}'.format(fileMeta.full_name, apiResp.code, apiResp.message))
return False
autoTagger.AutoTagAmbarFile(ambarFile)
return True
except Exception as e:
logger.LogMessage('error', 'error processing task {0}'.format(repr(e)))
return False