def processImage( documentId, features, bucketName, objectName, outputTableName, documentsTableName ): detectText = "Text" in features detectForms = "Forms" in features detectTables = "Tables" in features response = callTextract( bucketName, objectName, detectText, detectForms, detectTables ) dynamodb = AwsHelper().getResource("dynamodb") ddb = dynamodb.Table(outputTableName) print("Generating output for DocumentId: {}".format(documentId)) opg = OutputGenerator( documentId, response, bucketName, objectName, detectForms, detectTables, ddb ) opg.run() print("DocumentId: {}".format(documentId)) ds = datastore.DocumentStore(documentsTableName, outputTableName) ds.markDocumentComplete(documentId)
def processDocument(self, ips, i, document): print("\nTextracting Document # {}: {}".format(i, document)) print('=' * (len(document) + 30)) # Get document textracted dp = DocumentProcessor(ips["bucketName"], document, ips["awsRegion"], ips["text"], ips["forms"], ips["tables"]) response = dp.run() print("Recieved Textract response...") #FileHelper.writeToFile("temp-response.json", json.dumps(response)) #Generate output files print("Generating output...") name, ext = FileHelper.getFileNameAndExtension(document) opg = OutputGenerator( response, os.path.join(ips["output"], "{}-{}".format(name, ext)), ips["forms"], ips["tables"]) opg.run() if (ips["insights"] or ips["medical-insights"] or ips["translate"]): opg.generateInsights(ips["insights"], ips["medical-insights"], ips["translate"], ips["awsRegion"]) print("{} textracted successfully.".format(document))
def processImage(documentId, features, bucketName, outputBucketName, objectName, outputTableName, documentsTableName, elasticsearchDomain): detectText = "Text" in features detectForms = "Forms" in features detectTables = "Tables" in features response = callTextract(bucketName, objectName, detectText, detectForms, detectTables) dynamodb = AwsHelper().getResource("dynamodb") ddb = dynamodb.Table(outputTableName) outputPath = '{}{}/{}'.format(PUBLIC_PATH_S3_PREFIX, documentId, SERVICE_OUTPUT_PATH_S3_PREFIX) print("Generating output for DocumentId: {} and storing in {}".format( documentId, outputPath)) opg = OutputGenerator(documentId, response, outputBucketName, objectName, detectForms, detectTables, ddb, outputPath, elasticsearchDomain) opg_output = opg.run() generatePdf(documentId, bucketName, objectName, outputBucketName, outputPath) # generate Comprehend and ComprehendMedical entities in S3 comprehendOutputPath = "{}{}".format(outputPath, COMPREHEND_PATH_S3_PREFIX) print("Comprehend output path: " + comprehendOutputPath) maxPages = 100 comprehendClient = ComprehendHelper() responseDocumentName = "{}{}response.json".format(outputPath, TEXTRACT_PATH_S3_PREFIX) comprehendAndMedicalEntities = comprehendClient.processComprehend( outputBucketName, responseDocumentName, comprehendOutputPath, maxPages) # if Kendra is available then let it index the document # index the searchable pdf in Kendra if 'KENDRA_INDEX_ID' in os.environ: kendraClient = KendraHelper() fileName = os.path.basename(objectName).split(".")[0] fileExtension = os.path.basename(objectName).split(".")[1] outputDocumentName = "{}{}-searchable.pdf".format(outputPath, fileName) kendraClient.indexDocument(os.environ['KENDRA_INDEX_ID'], os.environ['KENDRA_ROLE_ARN'], outputBucketName, outputDocumentName, documentId, fileExtension) print("Processed Comprehend data for document: {}".format(documentId)) for key, val in opg_output[KVPAIRS].items(): if key not in comprehendAndMedicalEntities: comprehendAndMedicalEntities[key] = val else: comprehendAndMedicalEntities[key].add(val) opg.indexDocument(opg_output[DOCTEXT], comprehendAndMedicalEntities) ds = datastore.DocumentStore(documentsTableName, outputTableName) ds.markDocumentComplete(documentId)
def processRequest(request): output = "" print(request) jobId = request['jobId'] jobTag = request['jobTag'] jobStatus = request['jobStatus'] jobAPI = request['jobAPI'] bucketName = request['bucketName'] objectName = request['objectName'] outputTable = request["outputTable"] outputBucket = request["outputBucket"] documentsTable = request["documentsTable"] qUrl = request["elasticQueueUrl"] pages = getJobResults(jobAPI, jobId) print("Result pages recieved: {}".format(len(pages))) dynamodb = AwsHelper().getResource("dynamodb") ddb = dynamodb.Table(outputTable) detectForms = False detectTables = False if (jobAPI == "StartDocumentAnalysis"): detectForms = True detectTables = True opg = OutputGenerator(jobId, jobTag, pages, outputBucket, objectName, detectForms, detectTables, ddb) opg.run() print("DocumentId: {}".format(jobTag)) ds = datastore.DocumentStore(documentsTable, outputTable) ds.markDocumentComplete(jobTag, jobId) jsonMessage = { 'documentId': jobTag, 'jobId': jobId, 'bucketName': outputBucket, 'objectName': objectName } client = AwsHelper().getClient('sqs') postMessage(client, qUrl, jsonMessage) output = "Processed -> Document: {}, Object: {}/{} processed.".format( jobTag, bucketName, objectName) print(output) return {'statusCode': 200, 'body': output}
def processRequest(request): output = "" print(request) jobId = request['jobId'] jobTag = request['jobTag'] jobStatus = request['jobStatus'] jobAPI = request['jobAPI'] bucketName = request['bucketName'] outputBucketName = request['outputBucketName'] objectName = request['objectName'] outputTable = request["outputTable"] documentsTable = request["documentsTable"] elasticsearchDomain = request["elasticsearchDomain"] pages = getJobResults(jobAPI, jobId) print("Result pages recieved: {}".format(len(pages))) dynamodb = AwsHelper().getResource("dynamodb") ddb = dynamodb.Table(outputTable) detectForms = False detectTables = False if (jobAPI == "StartDocumentAnalysis"): detectForms = True detectTables = True dynamodb = AwsHelper().getResource('dynamodb') ddb = dynamodb.Table(outputTable) opg = OutputGenerator(jobTag, pages, outputBucketName, objectName, detectForms, detectTables, ddb, elasticsearchDomain) opg.run() generatePdf(jobTag, bucketName, objectName, outputBucketName) print("DocumentId: {}".format(jobTag)) ds = datastore.DocumentStore(documentsTable, outputTable) ds.markDocumentComplete(jobTag) output = "Processed -> Document: {}, Object: {}/{} processed.".format( jobTag, bucketName, objectName) print(output) return {'statusCode': 200, 'body': output}
def processImage(documentId, features, bucketName, outputBucketName, objectName, outputTableName, documentsTableName, elasticsearchDomain): detectText = "Text" in features detectForms = "Forms" in features detectTables = "Tables" in features response = callTextract(bucketName, objectName, detectText, detectForms, detectTables) dynamodb = AwsHelper().getResource("dynamodb") ddb = dynamodb.Table(outputTableName) outputPath = '{}{}/{}'.format(PUBLIC_PATH_S3_PREFIX, documentId, SERVICE_OUTPUT_PATH_S3_PREFIX) print("Generating output for DocumentId: {} and storing in {}".format( documentId, outputPath)) opg = OutputGenerator(documentId, response, outputBucketName, objectName, detectForms, detectTables, ddb, outputPath, elasticsearchDomain) opg_output = opg.run() generatePdf(documentId, bucketName, objectName, outputBucketName, outputPath) # generate Comprehend and ComprehendMedical entities in S3 comprehendOutputPath = "{}{}".format(outputPath, COMPREHEND_PATH_S3_PREFIX) print("Comprehend output path: " + comprehendOutputPath) maxPages = 100 comprehendClient = ComprehendHelper() responseDocumentName = "{}{}response.json".format(outputPath, TEXTRACT_PATH_S3_PREFIX) comprehendAndMedicalEntities = comprehendClient.processComprehend( outputBucketName, responseDocumentName, comprehendOutputPath, maxPages) print("DocumentId: {}".format(documentId)) print("Processed Comprehend data: {}".format(comprehendAndMedicalEntities)) for key, val in opg_output[KVPAIRS].items(): if key not in comprehendAndMedicalEntities: comprehendAndMedicalEntities[key] = val else: comprehendAndMedicalEntities[key].add(val) opg.indexDocument(opg_output[DOCTEXT], comprehendAndMedicalEntities) ds = datastore.DocumentStore(documentsTableName, outputTableName) ds.markDocumentComplete(documentId)
def processRequest(request): output = "" print(request) jobId = request["jobId"] jobTag = request["jobTag"] jobStatus = request["jobStatus"] jobAPI = request["jobAPI"] bucketName = request["bucketName"] objectName = request["objectName"] outputTable = request["outputTable"] documentsTable = request["documentsTable"] pages = getJobResults(jobAPI, jobId) print("Result pages recieved: {}".format(len(pages))) dynamodb = AwsHelper().getResource("dynamodb") ddb = dynamodb.Table(outputTable) detectForms = False detectTables = False if jobAPI == "StartDocumentAnalysis": detectForms = True detectTables = True dynamodb = AwsHelper().getResource("dynamodb") ddb = dynamodb.Table(outputTable) opg = OutputGenerator( jobTag, pages, bucketName, objectName, detectForms, detectTables, ddb ) opg.run() print("DocumentId: {}".format(jobTag)) ds = datastore.DocumentStore(documentsTable, outputTable) ds.markDocumentComplete(jobTag) output = "Processed -> Document: {}, Object: {}/{} processed.".format( jobTag, bucketName, objectName ) print(output) return {"statusCode": 200, "body": output}
def processImage(documentId, features, bucketName, outputBucketName, objectName, outputTableName, documentsTableName, elasticsearchDomain): detectText = "Text" in features detectForms = "Forms" in features detectTables = "Tables" in features response = callTextract(bucketName, objectName, detectText, detectForms, detectTables) dynamodb = AwsHelper().getResource("dynamodb") ddb = dynamodb.Table(outputTableName) print("Generating output for DocumentId: {}".format(documentId)) opg = OutputGenerator(documentId, response, outputBucketName, objectName, detectForms, detectTables, ddb, elasticsearchDomain) opg_output = opg.run() generatePdf(documentId, bucketName, objectName, outputBucketName) # generate Comprehend and ComprehendMedical entities in S3 path = objectName + "-analysis" + "/" + documentId + "/" print("path: " + path) maxPages = 100 comprehendClient = ComprehendHelper() comprehendAndMedicalEntities = comprehendClient.processComprehend( outputBucketName, 'response.json', path, maxPages) print("DocumentId: {}".format(documentId)) print("Processed Comprehend data: {}".format(comprehendAndMedicalEntities)) for key, val in opg_output[KVPAIRS].items(): if key not in comprehendAndMedicalEntities: comprehendAndMedicalEntities[key] = val else: comprehendAndMedicalEntities[key].add(val) opg.indexDocument(opg_output[DOCTEXT], comprehendAndMedicalEntities) ds = datastore.DocumentStore(documentsTableName, outputTableName) ds.markDocumentComplete(documentId)
def processImage(features, bucket_name, object_name): detectTables = "detectTables" in features detectText = "detectText" in features response = callTextract(bucket_name, object_name, detectText, detectTables) print("Generating output for DocumentId: {}".format(object_name)) #print(json.dumps(response)) #get page metadata metadata = get_page_metadata(bucket_name=bucket_name, object_name=object_name) opg = OutputGenerator(response, bucketName=bucket_name, objectName=object_name, tables=detectTables, metadata=metadata) output = opg.run() print("DocumentId: {}".format(object_name)) return output
def processImage(documentId, bucketName, objectName, callerId): response = callTextract(bucketName, objectName) print("Generating output for documentId: {}".format(documentId)) opg = OutputGenerator(documentId=documentId, response=response, bucketName=textractBucketName, objectName=objectName, forms=False, tables=False) tagging = "documentId={}".format(documentId) opg.writeTextractOutputs(taggingStr=tagging) lineage_client.recordLineage({ "documentId": documentId, "callerId": callerId, "sourceBucketName": bucketName, "targetBucketName": textractBucketName, "sourceFileName": objectName, "targetFileName": objectName })
def generateOutput(filePath, response): print("Generating output...") name, ext = FileHelper.getFileNameAndExtension(filePath) opg = OutputGenerator(response, "{}-v2-{}".format(name, ext), True, True) opg.run() opg.generateInsights(True, True, 'es', 'us-east-1')
def processRequest(request): output = "" print(request) jobId = request['jobId'] jobTag = request['jobTag'] jobStatus = request['jobStatus'] jobAPI = request['jobAPI'] bucketName = request['bucketName'] outputBucketName = request['outputBucketName'] objectName = request['objectName'] outputTable = request["outputTable"] documentsTable = request["documentsTable"] elasticsearchDomain = request["elasticsearchDomain"] pages = getJobResults(jobAPI, jobId) print("Result pages recieved: {}".format(len(pages))) dynamodb = AwsHelper().getResource("dynamodb") ddb = dynamodb.Table(outputTable) detectForms = False detectTables = False if (jobAPI == "StartDocumentAnalysis"): detectForms = True detectTables = True dynamodb = AwsHelper().getResource('dynamodb') ddb = dynamodb.Table(outputTable) opg = OutputGenerator(jobTag, pages, outputBucketName, objectName, detectForms, detectTables, ddb, elasticsearchDomain) opg_output = opg.run() generatePdf(jobTag, bucketName, objectName, outputBucketName) # generate Comprehend and ComprehendMedical entities path = objectName + "-analysis" + "/" + jobTag + "/" print("path: " + path) maxPages = 100 comprehendClient = ComprehendHelper() comprehendAndMedicalEntities = comprehendClient.processComprehend( outputBucketName, 'response.json', path, maxPages) print("DocumentId: {}".format(jobTag)) # index document once the comprehend entities and KVPairs have been extracted for key, val in opg_output[KVPAIRS].items(): if key not in comprehendAndMedicalEntities: comprehendAndMedicalEntities[key] = val else: comprehendAndMedicalEntities[key].add(val) opg.indexDocument(opg_output[DOCTEXT], comprehendAndMedicalEntities) ds = datastore.DocumentStore(documentsTable, outputTable) ds.markDocumentComplete(jobTag) output = "Processed -> Document: {}, Object: {}/{} processed.".format( jobTag, bucketName, objectName) return {'statusCode': 200, 'body': output}
def processRequest(request): output = "" print("Request : {}".format(request)) jobId = request['jobId'] documentId = request['jobTag'] jobStatus = request['jobStatus'] jobAPI = request['jobAPI'] bucketName = request['bucketName'] outputBucketName = request['outputBucketName'] objectName = request['objectName'] outputTable = request["outputTable"] documentsTable = request["documentsTable"] elasticsearchDomain = request["elasticsearchDomain"] pages = getJobResults(jobAPI, jobId) print("Result pages recieved: {}".format(len(pages))) dynamodb = AwsHelper().getResource("dynamodb") ddb = dynamodb.Table(outputTable) detectForms = False detectTables = False if (jobAPI == "StartDocumentAnalysis"): detectForms = True detectTables = True dynamodb = AwsHelper().getResource('dynamodb') ddb = dynamodb.Table(outputTable) outputPath = '{}{}/{}'.format(PUBLIC_PATH_S3_PREFIX, documentId, SERVICE_OUTPUT_PATH_S3_PREFIX) print("Generating output for DocumentId: {} and storing in {}".format( documentId, outputPath)) opg = OutputGenerator(documentId, pages, outputBucketName, objectName, detectForms, detectTables, ddb, outputPath, elasticsearchDomain) opg_output = opg.run() generatePdf(documentId, bucketName, objectName, outputBucketName, outputPath) # generate Comprehend and ComprehendMedical entities comprehendOutputPath = "{}{}".format(outputPath, COMPREHEND_PATH_S3_PREFIX) print("Comprehend output path: " + comprehendOutputPath) maxPages = 100 comprehendClient = ComprehendHelper() responseDocumentName = "{}{}response.json".format(outputPath, TEXTRACT_PATH_S3_PREFIX) comprehendAndMedicalEntities = comprehendClient.processComprehend( outputBucketName, responseDocumentName, comprehendOutputPath, maxPages) # if Kendra is available then let it index the document if 'KENDRA_INDEX_ID' in os.environ: kendraClient = KendraHelper() fileName = os.path.basename(objectName).split(".")[0] fileExtension = os.path.basename(objectName).split(".")[1] outputDocumentName = "{}{}-searchable.pdf".format(outputPath, fileName) kendraClient.indexDocument(os.environ['KENDRA_INDEX_ID'], os.environ['KENDRA_ROLE_ARN'], bucketName, outputDocumentName, documentId, fileExtension) print("DocumentId: {}".format(documentId)) print("Processed Comprehend data: {}".format(comprehendAndMedicalEntities)) # index document once the comprehend entities and KVPairs have been extracted for key, val in opg_output[KVPAIRS].items(): if key not in comprehendAndMedicalEntities: comprehendAndMedicalEntities[key] = val else: comprehendAndMedicalEntities[key].add(val) opg.indexDocument(opg_output[DOCTEXT], comprehendAndMedicalEntities) ds = datastore.DocumentStore(documentsTable, outputTable) ds.markDocumentComplete(documentId) output = "Processed -> Document: {}, Object: {}/{} processed.".format( documentId, bucketName, objectName) return {'statusCode': 200, 'body': output}
def runComprehend(bucketName, objectName, callerId): comprehend = AwsHelper().getClient('comprehend') documentId, documentName = dissectObjectName(objectName) assert ( documentId == S3Helper().getTagsS3(bucketName, objectName).get('documentId', None) ), "File path {} does not match the expected documentId tag of the object triggered.".format( objectName) textractOutputJson = json.loads(S3Helper().readFromS3( bucketName, objectName)) og = OutputGenerator(response=textractOutputJson, forms=False, tables=False) pipeline_client.body = { "documentId": documentId, "bucketName": bucketName, "objectName": objectName, "stage": PIPELINE_STAGE } pipeline_client.stageInProgress() document = Document(textractOutputJson) originalFileName = "{}/{}".format(documentId, documentName) comprehendFileName = originalFileName + "/comprehend-output.json" comprehendFileS3Url = "https://{}.s3.amazonaws.com/{}".format( comprehendBucket, urllib.parse.quote_plus(comprehendFileName, safe="/")) tagging = "documentId={}".format(documentId) es.connect() esPayload = [] page_num = 1 for page in document.pages: table = og.structurePageTable(page) forms = og.structurePageForm(page) text = og.structurePageText(page) keyPhrases = [] entitiesDetected = {} lenOfEncodedText = len(text) print("Comprehend documentId {} processing page {}".format( documentId, str(page_num))) print("Length of encoded text is " + str(lenOfEncodedText)) if lenOfEncodedText > COMPREHEND_CHARACTER_LIMIT: print( "Size was too big to run singularly; breaking up the page text into chunks" ) try: chunksOfText = chunkUpTheText(text) except Exception as e: pipeline_client.stageFailed( "Could not determine how to snip the text on page {} into chunks." .format(page_num)) raise (e) keyPhrases, entitiesDetected = batchSendToComprehend( comprehend, chunksOfText, 'en') else: keyPhrases, entitiesDetected = singularSendToComprehend( comprehend, text, 'en') esPageLoad = compileESPayload(es, page_num, keyPhrases, entitiesDetected, text, table, forms, documentId) esPayload.append(esPageLoad) page_num = page_num + 1 try: es.post_bulk(index=esIndex, payload=esPayload) except Exception as e: pipeline_client.stageFailed("Could not post to Elasticsearch") raise (e) print("Data uploaded to ES") try: S3Helper().writeToS3(json.dumps(esPayload), comprehendBucket, comprehendFileName, taggingStr=tagging) except Exception as e: pipeline_client.stageFailed("Failed to write comprehend payload to S3") raise (e) lineage_client.recordLineage({ "documentId": documentId, "callerId": callerId, "sourceBucketName": bucketName, "targetBucketName": comprehendBucket, "sourceFileName": objectName, "targetFileName": comprehendFileName }) pipeline_client.stageSucceeded() print("Comprehend data uploaded to S3 at {}".format(comprehendFileName))
def processRequest(request): output = "" status = request['jobStatus'] jobId = request['jobId'] jobTag = request['jobTag'] jobAPI = request['jobAPI'] bucketName = request['bucketName'] objectName = request['objectName'] pipeline_client.body = { "documentId": jobTag, "bucketName": bucketName, "objectName": objectName, "stage": PIPELINE_STAGE } if status == 'FAILED': pipeline_client.stageFailed( "Textract Analysis didn't complete successfully") raise Exception( "Textract job for document ID {}; bucketName {} fileName {}; failed during Textract analysis. Please double check the document quality" .format(jobTag, bucketName, objectName)) pipeline_client.stageInProgress() try: pages = getJobResults(jobAPI, jobId) except Exception as e: pipeline_client.stageFailed() raise (e) print("Result pages received: {}".format(len(pages))) detectForms = False detectTables = False if (jobAPI == "StartDocumentAnalysis"): detectForms = True detectTables = True try: opg = OutputGenerator(documentId=jobTag, response=pages, bucketName=textractBucketName, objectName=objectName, forms=detectForms, tables=detectTables) except Exception as e: pipeline_client.stageFailed( "Could not convert results from Textract into processable object. Try uploading again." ) raise (e) tagging = "documentId={}".format(jobTag) opg.writeTextractOutputs(taggingStr=tagging) lineage_client.recordLineage({ "documentId": jobTag, "callerId": request["callerId"], "sourceBucketName": bucketName, "targetBucketName": textractBucketName, "sourceFileName": objectName, "targetFileName": objectName }) output = "Processed -> Document: {}, Object: {}/{} processed.".format( jobTag, bucketName, objectName) pipeline_client.stageSucceeded() print(output) return {'statusCode': 200, 'body': output}