def main(input_filename, output_filename, correct_length): pdf_directory, pages = split(input_filename) image_directory, images = convert_to_images(pages) pages_with_images = zip(pages, images) docs = split_documents(pages_with_images, correct_length) # split into docs with and without padding good_docs = [doc for doc in docs if not doc.isPadded] padded_docs = [doc for doc in docs if doc.isPadded] # flatten and pull out just the pdf filenames good_pdfs = [pdf for doc in good_docs for pdf in doc.pdf_pages] padded_pdfs = [pdf for doc in padded_docs for pdf in doc.pdf_pages] if len(good_pdfs) > 0: pypdftk.concat(good_pdfs, output_filename + '_good.pdf') if len(padded_pdfs) > 0: pypdftk.concat(padded_pdfs, output_filename + '_padded.pdf') # cleanup temp files for pdf_name, image_name in pages_with_images: os.remove(pdf_name) os.remove(image_name) os.rmdir(pdf_directory) os.rmdir(image_directory) show_summary(good_docs, padded_docs) print("Merged results written to {0}_good.pdf and {0}_padded.pdf".format( output_filename)) return 0
def run(idir, bdir, ofile): authors, venues, papers = read_all_info(idir) fpdf_names = [] tmpdirname = tempfile.mkdtemp() for p in papers: if p['pubTypeSlot'] == 'Conference' or p['pubTypeSlot'] == 'Journal': if 'pdfLink' not in p: print("pdfLink missing:", p['id']) elif p['pdfLink'].startswith("http"): print("local link missing:", p['id']) else: source = bdir + "/" + p['pdfLink'] i = len(fpdf_names) dest = "%s/%d.pdf" % (tmpdirname, i) print("getting %s, putting it %s" % (source, dest)) get_pdf(source, dest) tdir = "%s/%d/" % (tmpdirname, i) os.mkdir(tdir) fpdf_names.append(tdir + "page_01.pdf") pypdftk.split(dest, tdir) pypdftk.concat(fpdf_names, out_file=ofile) shutil.rmtree(tmpdirname)
def process(config): """ parse the JSON configuration then process each item independently """ tmp_dir = tempfile.mkdtemp(prefix="pdfFactory-") callback_url = config.get("callback", None) if not os.path.lexists(config["output"]) or config.get("overwrite", True): try: merge_list = [] for item in config["items"]: item_data = {} if "data" in config: item_data.update(config["data"]) if "data" in item: item_data.update(item["data"]) item["data"] = item_data merge_list.append(process_item(item, tmp_dir)) except KeyError as e: log.error("Error when parsing JSON file, some important values are missing !") log.error("Missing value: %s", e) clean_failure(tmp_dir, callback_url) except: log.error("Cannot proceed, got an unknown error:\n %s\n", traceback.format_exc()) clean_failure(tmp_dir, callback_url) # Merge resulting PDF log.debug("Merging pdf: %s...", str(merge_list)) # So A/foo/../B don't become A/ output = os.path.join(BASE_OUTPUT_DIR, config["output"]) check_output_folder(BASE_OUTPUT_DIR, output, create_folder=True) pypdftk.concat(merge_list, output) else: log.info("Document '%s' already exists and do not need re-generation.", config["output"]) success_callback(True, callback_url) clean_tmp(tmp_dir)
def test_concat(): files = ["./Out/page_01.pdf", "./Out/page_02.pdf", "./Out/page_03.pdf"] pypdftk.concat(files, "./Out/sample_output.pdf")
def print_sc1_line( f3x_data, md5_directory, sc1, sc1_start_page, total_no_of_pages, image_num=None ): sc1_start_page += 1 # is_memo_page = 0 sc1_schedule_page_dict = {} if image_num: sc1_schedule_page_dict["IMGNO"] = image_num image_num += 1 sc1_schedule_page_dict["PAGENO"] = sc1_start_page sc1_schedule_page_dict["TRANSACTION_ID"] = sc1.get("transactionId") sc1_schedule_page_dict["TOTALPAGES"] = total_no_of_pages sc1_schedule_page_dict["committeeName"] = f3x_data.get("committeeName") sc1_schedule_page_dict["committeeId"] = f3x_data.get("committeeId") sc1_schedule_page_dict["lenderName"] = sc1.get("lenderOrganizationName") for i in [ "lenderStreet1", "lenderStreet2", "lenderCity", "lenderState", "lenderZipCode", "loanInterestRate", "isLoanRestructured", "otherPartiesLiable", "pledgedCollateralIndicator", "pledgeCollateralDescription", "perfectedInterestIndicator", "futureIncomeIndicator", "SCPageNo", ]: sc1_schedule_page_dict[i] = sc1.get(i) if sc1.get("loanIncurredDate") != "": date_array = sc1.get("loanIncurredDate").split("/") sc1_schedule_page_dict["loanIncurredDateMonth"] = date_array[0] sc1_schedule_page_dict["loanIncurredDateDay"] = date_array[1] sc1_schedule_page_dict["loanIncurredDateYear"] = date_array[2] if sc1.get("loanDueDate") not in ["none", "null", " ", "", None]: if "-" in sc1.get("loanDueDate"): date_array = sc1.get("loanDueDate").split("-") if len(date_array) == 3: sc1_schedule_page_dict["loanDueDateMonth"] = date_array[1] sc1_schedule_page_dict["loanDueDateDay"] = date_array[2] sc1_schedule_page_dict["loanDueDateYear"] = date_array[0] else: sc1_schedule_page_dict["loanDueDateYear"] = sc1.get("loanDueDate") elif "/" in sc1.get("loanDueDate"): date_array = sc1.get("loanDueDate").split("/") if len(date_array) == 3: sc1_schedule_page_dict["loanDueDateMonth"] = date_array[0] sc1_schedule_page_dict["loanDueDateDay"] = date_array[1] sc1_schedule_page_dict["loanDueDateYear"] = date_array[2] else: sc1_schedule_page_dict["loanDueDateYear"] = sc1.get("loanDueDate") else: sc1_schedule_page_dict["loanDueDateYear"] = sc1.get("loanDueDate") if sc1.get("originalLoanDate") != "": date_array = sc1.get("originalLoanDate").split("/") sc1_schedule_page_dict["originalLoanDateMonth"] = date_array[0] sc1_schedule_page_dict["originalLoanDateDay"] = date_array[1] sc1_schedule_page_dict["originalLoanDateYear"] = date_array[2] if sc1.get("depositoryAccountEstablishedDate") != "": date_array = sc1.get("depositoryAccountEstablishedDate").split("/") sc1_schedule_page_dict["ACCOUNT_EST_DATE_MM"] = date_array[0] sc1_schedule_page_dict["ACCOUNT_EST_DATE_DD"] = date_array[1] sc1_schedule_page_dict["ACCOUNT_EST_DATE_YY"] = date_array[2] sc1_schedule_page_dict["loanAmount"] = "{0:.2f}".format( float(sc1.get("loanAmount")) ) sc1_schedule_page_dict["creditAmountThisDraw"] = "{0:.2f}".format( float(sc1.get("creditAmountThisDraw")) ) sc1_schedule_page_dict["totalOutstandingBalance"] = "{0:.2f}".format( float(sc1.get("totalOutstandingBalance")) ) sc1_schedule_page_dict["BACK_REF_TRAN_ID"] = sc1.get( "backReferenceTransactionIdNumber" ) sc1_schedule_page_dict["pledgeCollateralAmount"] = "{0:.2f}".format( float(sc1.get("pledgeCollateralAmount")) ) sc1_schedule_page_dict["PLEDGE_DESC"] = sc1.get("futureIncomeDescription") sc1_schedule_page_dict["PLEDGE_ESTIMATED_AMOUNT"] = "{0:.2f}".format( float(sc1.get("futureIncomeEstimate")) ) treasurerName = "" for i in [ "treasurerPrefix", "treasurerLastName", "treasurerFirstName", "treasurerMiddleName", "treasurerSuffix", ]: if sc1.get(i) != "": treasurerName += sc1.get(i) + " " sc1_schedule_page_dict["COMMITTEE_TREASURER_NAME"] = treasurerName[0:-1] sc1_schedule_page_dict["DEPOSITORY_NAME"] = sc1.get("depositoryAccountLocation") sc1_schedule_page_dict["DEPOSITORY_STREET1"] = sc1.get("depositoryAccountStreet1") sc1_schedule_page_dict["DEPOSITORY_STREET2"] = sc1.get("depositoryAccountStreet2") sc1_schedule_page_dict["DEPOSITORY_CITY"] = sc1.get("depositoryAccountCity") sc1_schedule_page_dict["DEPOSITORY_STATE"] = sc1.get("depositoryAccountState") sc1_schedule_page_dict["DEPOSITORY_ZIP"] = sc1.get("depositoryAccountZipCode") sc1_schedule_page_dict["BASIS"] = sc1.get("basisOfLoanDescription") if sc1.get("treasurerSignedDate") != "": date_array = sc1.get("treasurerSignedDate").split("/") sc1_schedule_page_dict["TREASUER_SIGN_DATE_MM"] = date_array[0] sc1_schedule_page_dict["TREASUER_SIGN_DATE_DD"] = date_array[1] sc1_schedule_page_dict["TREASUER_SIGN_DATE_YY"] = date_array[2] authorizedName = "" for i in [ "authorizedPrefix", "authorizedLastName", "authorizedFirstName", "authorizedMiddleName", "authorizedSuffix", ]: if sc1.get(i) != "": authorizedName += sc1.get(i) + " " sc1_schedule_page_dict["AUTH_REP_NAME"] = authorizedName[0:-1] sc1_schedule_page_dict["AUTH_REP_TITLE"] = sc1.get("authorizedTitle") if sc1.get("authorizedSignedDate") != "": date_array = sc1.get("authorizedSignedDate").split("/") sc1_schedule_page_dict["AUTH_REP_SIGN_MM"] = date_array[0] sc1_schedule_page_dict["AUTH_REP_SIGN_DD"] = date_array[1] sc1_schedule_page_dict["AUTH_REP_SIGN_YY"] = date_array[2] sc1_infile = current_app.config["FORM_TEMPLATES_LOCATION"].format("SC1") sc1_outfile = md5_directory + "SC" + "/page_" + str(sc1_start_page) + ".pdf" pypdftk.fill_form(sc1_infile, sc1_schedule_page_dict, sc1_outfile) # Memo text changes if sc1_schedule_page_dict.get("memoDescription"): # is_memo_page = 1 memo_dict = {} temp_memo_outfile = md5_directory + "SC/page_memo_temp.pdf" memo_infile = current_app.config["FORM_TEMPLATES_LOCATION"].format("TEXT") memo_outfile = md5_directory + "SC/page_memo_" + str(sc1_start_page) + ".pdf" memo_dict["scheduleName_1"] = "SC1" memo_dict["memoDescription_1"] = sc1_schedule_page_dict["memoDescription"] memo_dict["PAGESTR"] = ( "PAGE " + str(sc1_start_page + 1) + " / " + str(total_no_of_pages) ) if ( "transactionId" in sc1_schedule_page_dict and sc1_schedule_page_dict["transactionId"] ): memo_dict["transactionId_1"] = sc1_schedule_page_dict["transactionId"] if image_num: memo_dict["IMGNO"] = image_num image_num += 1 # build page pypdftk.fill_form(memo_infile, memo_dict, memo_outfile) pypdftk.concat([sc1_outfile, memo_outfile], temp_memo_outfile) os.remove(memo_outfile) os.rename(temp_memo_outfile, sc1_outfile) if path.isfile(md5_directory + "SC/all_pages.pdf"): pypdftk.concat( [ md5_directory + "SC/all_pages.pdf", md5_directory + "SC" + "/page_" + str(sc1_start_page) + ".pdf", ], md5_directory + "SC/temp_all_pages.pdf", ) os.rename( md5_directory + "SC/temp_all_pages.pdf", md5_directory + "SC/all_pages.pdf" ) else: os.rename( md5_directory + "SC/all_pages.pdf", md5_directory + "SC" + "/page_" + str(sc1_start_page) + ".pdf", ) return image_num
def print_sc_line( f3x_data, md5_directory, sc_schedules, sc_start_page, total_no_of_pages, image_num=None, ): try: sc_schedule_total = 0 os.makedirs(md5_directory + "SC/", exist_ok=True) sc_infile = current_app.config["FORM_TEMPLATES_LOCATION"].format("SC") sc1_list = [] totalOutstandingLoans = "0.00" page_num = 0 for sc in sc_schedules: page_subtotal = "{0:.2f}".format(float(sc.get("loanBalance"))) # memo_array = [] sc_schedule_total += float(page_subtotal) sc_schedule_page_dict = {} sc_schedule_page_dict["TRANSACTION_ID"] = sc.get("transactionId") sc_schedule_page_dict["totalPages"] = total_no_of_pages sc_schedule_page_dict["committeeName"] = f3x_data.get("committeeName") sc_schedule_page_dict["pageSubtotal"] = page_subtotal for item in [ "memoCode", "memoDescription", "lenderStreet1", "lenderStreet2", "lenderCity", "lenderState", "lenderZipCode", "electionOtherDescription", "isLoanSecured", ]: sc_schedule_page_dict[item] = sc.get(item) for item in [ "loanAmountOriginal", "loanPaymentToDate", "loanBalance", "loanInterestRate", ]: sc_schedule_page_dict[item] = "{0:.2f}".format(float(sc.get(item))) if "electionCode" in sc and sc.get("electionCode") != "": sc_schedule_page_dict["electionType"] = sc.get("electionCode")[0:1] sc_schedule_page_dict["electionYear"] = sc.get("electionCode")[1:5] # if sc.get('lenderOrganizationName') == "": if not sc.get("lenderOrganizationName"): lenderName = "" for item in [ "lenderLastName", "lenderFirstName", "lenderMiddleName", "lenderPrefix", "lenderSuffix", ]: if sc.get(item): lenderName += sc.get(item) + " " sc_schedule_page_dict["lenderName"] = lenderName[0:-1] else: sc_schedule_page_dict["lenderName"] = sc.get("lenderOrganizationName") if sc.get("loanIncurredDate"): date_array = sc.get("loanIncurredDate").split("/") sc_schedule_page_dict["loanIncurredDateMonth"] = date_array[0] sc_schedule_page_dict["loanIncurredDateDay"] = date_array[1] sc_schedule_page_dict["loanIncurredDateYear"] = date_array[2] if sc.get("loanDueDate"): if "-" in sc.get("loanDueDate"): date_array = sc.get("loanDueDate").split("-") if len(date_array) == 3: sc_schedule_page_dict["loanDueDateMonth"] = date_array[1] sc_schedule_page_dict["loanDueDateDay"] = date_array[2] sc_schedule_page_dict["loanDueDateYear"] = date_array[0] else: sc_schedule_page_dict["loanDueDateYear"] = sc.get("loanDueDate") elif "/" in sc.get("loanDueDate"): date_array = sc.get("loanDueDate").split("/") if len(date_array) == 3: sc_schedule_page_dict["loanDueDateMonth"] = date_array[0] sc_schedule_page_dict["loanDueDateDay"] = date_array[1] sc_schedule_page_dict["loanDueDateYear"] = date_array[2] else: sc_schedule_page_dict["loanDueDateYear"] = sc.get("loanDueDate") else: sc_schedule_page_dict["loanDueDateYear"] = sc.get("loanDueDate") if sc.get("child"): sc2 = [] for sc_child in sc.get("child"): if sc_child.get("transactionTypeIdentifier") == "SC2": sc2.append(sc_child) elif sc_child.get("transactionTypeIdentifier") == "SC1": # sc_child['SCPageNo'] = sc_start_page sc1_list.append(sc_child) if sc2: sc2_list = [] temp_sc2 = [] for i in range(len(sc2)): temp_sc2.append(sc2[i]) if i % 4 == 3 or i == len(sc2) - 1: sc2_list.append(temp_sc2) temp_sc2 = [] for i in range(len(sc2_list)): sc_schedule_single_page_dict = {} sc_schedule_single_page_dict = sc_schedule_page_dict for j in range(len(sc2_list[i])): sc2_name = "" for k in [ "prefix", "lastName", "firstName", "middleName", "suffix", ]: if sc2_list[i][j].get(k) != "": sc2_name += sc2_list[i][j].get(k) + "," sc_schedule_single_page_dict[ "name_{}".format(j + 1) ] = sc2_name[0:-1] sc_schedule_single_page_dict[ "street1_{}".format(j + 1) ] = sc2_list[i][j].get("street1") sc_schedule_single_page_dict[ "street2_{}".format(j + 1) ] = sc2_list[i][j].get("street2") sc_schedule_single_page_dict[ "city_{}".format(j + 1) ] = sc2_list[i][j].get("city") sc_schedule_single_page_dict[ "state_{}".format(j + 1) ] = sc2_list[i][j].get("state") sc_schedule_single_page_dict[ "zipCode_{}".format(j + 1) ] = sc2_list[i][j].get("zipCode") sc_schedule_single_page_dict[ "employer_{}".format(j + 1) ] = sc2_list[i][j].get("employer") sc_schedule_single_page_dict[ "occupation_{}".format(j + 1) ] = sc2_list[i][j].get("occupation") sc_schedule_single_page_dict[ "guaranteedAmount_{}".format(j + 1) ] = "{0:.2f}".format( float(sc2_list[i][j].get("guaranteedAmount")) ) sc_schedule_single_page_dict["pageNo"] = ( sc_start_page + page_num ) page_num += 1 if image_num: sc_schedule_single_page_dict["IMGNO"] = image_num image_num += 1 if ( sc_schedules[len(sc_schedules) - 1].get("transactionId") == sc_schedule_single_page_dict.get("TRANSACTION_ID") and i == len(sc2_list) - 1 ): totalOutstandingLoans = sc_schedule_single_page_dict[ "scheduleTotal" ] = "{0:.2f}".format(sc_schedule_total) sc_outfile = ( md5_directory + "SC" + "/page_" + str(sc_start_page) + ".pdf" ) pypdftk.fill_form( sc_infile, sc_schedule_single_page_dict, sc_outfile ) # Memo text changes if sc_schedule_page_dict.get("memoDescription"): memo_dict = {} temp_memo_outfile = md5_directory + "SC/page_memo_temp.pdf" memo_infile = current_app.config[ "FORM_TEMPLATES_LOCATION" ].format("TEXT") memo_outfile = ( md5_directory + "SC/page_memo_" + str(sc_start_page) + ".pdf" ) memo_dict["scheduleName_1"] = "SC13" memo_dict["PAGESTR"] = ( "PAGE " + str(sc_start_page + page_num) + " / " + str(total_no_of_pages) ) memo_dict["memoDescription_1"] = sc_schedule_page_dict[ "memoDescription" ] if ( "TRANSACTION_ID" in sc_schedule_page_dict and sc_schedule_page_dict["TRANSACTION_ID"] ): memo_dict["transactionId_1"] = sc_schedule_page_dict[ "TRANSACTION_ID" ] page_num += 1 if image_num: memo_dict["IMGNO"] = image_num image_num += 1 # build memo page pypdftk.fill_form(memo_infile, memo_dict, memo_outfile) pypdftk.concat( [sc_outfile, memo_outfile], temp_memo_outfile ) os.remove(memo_outfile) os.rename(temp_memo_outfile, sc_outfile) for j in range(len(sc2_list[i])): del sc_schedule_single_page_dict["name{}".format(j + 1)] del sc_schedule_single_page_dict["street1_{}".format(j + 1)] del sc_schedule_single_page_dict["street2_{}".format(j + 1)] del sc_schedule_single_page_dict["city_{}".format(j + 1)] del sc_schedule_single_page_dict["state_{}".format(j + 1)] del sc_schedule_single_page_dict["zipCode_{}".format(j + 1)] del sc_schedule_single_page_dict[ "employer_{}".format(j + 1) ] del sc_schedule_single_page_dict[ "occupation_{}".format(j + 1) ] del sc_schedule_single_page_dict[ "guaranteedAmount_{}".format(j + 1) ] if path.isfile(md5_directory + "SC/all_pages.pdf"): pypdftk.concat( [ md5_directory + "SC/all_pages.pdf", md5_directory + "SC" + "/page_" + str(sc_start_page) + ".pdf", ], md5_directory + "SC/temp_all_pages.pdf", ) os.rename( md5_directory + "SC/temp_all_pages.pdf", md5_directory + "SC/all_pages.pdf", ) else: os.rename( md5_directory + "SC" + "/page_" + str(sc_start_page) + ".pdf", md5_directory + "SC/all_pages.pdf", ) else: sc_schedule_page_dict["pageNo"] = sc_start_page + page_num page_num += 1 if image_num: sc_schedule_page_dict["IMGNO"] = image_num image_num += 1 if sc_schedules[len(sc_schedules) - 1].get( "transactionId" ) == sc_schedule_page_dict.get("TRANSACTION_ID"): totalOutstandingLoans = sc_schedule_page_dict[ "scheduleTotal" ] = "{0:.2f}".format(sc_schedule_total) sc_outfile = ( md5_directory + "SC" + "/page_" + str(sc_start_page) + ".pdf" ) pypdftk.fill_form(sc_infile, sc_schedule_page_dict, sc_outfile) # Memo text changes if sc_schedule_page_dict.get("memoDescription"): memo_dict = {} temp_memo_outfile = md5_directory + "SC/page_memo_temp.pdf" memo_infile = current_app.config[ "FORM_TEMPLATES_LOCATION" ].format("TEXT") memo_outfile = ( md5_directory + "SC/page_memo_" + str(sc_start_page) + ".pdf" ) memo_dict["scheduleName_1"] = "SC13" memo_dict["PAGESTR"] = ( "PAGE " + str(sc_start_page + page_num) + " / " + str(total_no_of_pages) ) memo_dict["memoDescription_1"] = sc_schedule_page_dict[ "memoDescription" ] if ( "TRANSACTION_ID" in sc_schedule_page_dict and sc_schedule_page_dict["TRANSACTION_ID"] ): memo_dict["transactionId_1"] = sc_schedule_page_dict[ "TRANSACTION_ID" ] page_num += 1 if image_num: memo_dict["IMGNO"] = image_num image_num += 1 # build page pypdftk.fill_form(memo_infile, memo_dict, memo_outfile) pypdftk.concat([sc_outfile, memo_outfile], temp_memo_outfile) os.remove(memo_outfile) os.rename(temp_memo_outfile, sc_outfile) if path.isfile(md5_directory + "SC/all_pages.pdf"): pypdftk.concat( [ md5_directory + "SC/all_pages.pdf", md5_directory + "SC" + "/page_" + str(sc_start_page) + ".pdf", ], md5_directory + "SC/temp_all_pages.pdf", ) os.rename( md5_directory + "SC/temp_all_pages.pdf", md5_directory + "SC/all_pages.pdf", ) else: os.rename( md5_directory + "SC" + "/page_" + str(sc_start_page) + ".pdf", md5_directory + "SC/all_pages.pdf", ) else: sc_schedule_page_dict["pageNo"] = sc_start_page + page_num page_num += 1 if image_num: sc_schedule_page_dict["IMGNO"] = image_num image_num += 1 if sc_schedules[len(sc_schedules) - 1].get( "transactionId" ) == sc_schedule_page_dict.get("TRANSACTION_ID"): totalOutstandingLoans = sc_schedule_page_dict[ "scheduleTotal" ] = "{0:.2f}".format(sc_schedule_total) sc_outfile = ( md5_directory + "SC" + "/page_" + str(sc_start_page) + ".pdf" ) pypdftk.fill_form(sc_infile, sc_schedule_page_dict, sc_outfile) # Memo text changes if sc_schedule_page_dict.get("memoDescription"): memo_dict = {} temp_memo_outfile = md5_directory + "SC/page_memo_temp.pdf" memo_infile = current_app.config["FORM_TEMPLATES_LOCATION"].format( "TEXT" ) memo_outfile = ( md5_directory + "SC/page_memo_" + str(sc_start_page) + ".pdf" ) memo_dict["scheduleName_1"] = "SC13" memo_dict["PAGESTR"] = ( "PAGE " + str(sc_start_page + page_num) + " / " + str(total_no_of_pages) ) memo_dict["memoDescription_1"] = sc_schedule_page_dict[ "memoDescription" ] if ( "TRANSACTION_ID" in sc_schedule_page_dict and sc_schedule_page_dict["TRANSACTION_ID"] ): memo_dict["transactionId_1"] = sc_schedule_page_dict[ "TRANSACTION_ID" ] page_num += 1 if image_num: memo_dict["IMGNO"] = image_num image_num += 1 # build page pypdftk.fill_form(memo_infile, memo_dict, memo_outfile) pypdftk.concat([sc_outfile, memo_outfile], temp_memo_outfile) os.remove(memo_outfile) os.rename(temp_memo_outfile, sc_outfile) if path.isfile(md5_directory + "SC/all_pages.pdf"): pypdftk.concat( [ md5_directory + "SC/all_pages.pdf", md5_directory + "SC" + "/page_" + str(sc_start_page) + ".pdf", ], md5_directory + "SC/temp_all_pages.pdf", ) os.rename( md5_directory + "SC/temp_all_pages.pdf", md5_directory + "SC/all_pages.pdf", ) else: os.rename( md5_directory + "SC" + "/page_" + str(sc_start_page) + ".pdf", md5_directory + "SC/all_pages.pdf", ) return sc1_list, sc_start_page + page_num - 1, totalOutstandingLoans, image_num except: # printing stack trace traceback.print_exception(*sys.exc_info())
def test_concat(): files = ["./Out/page_01.pdf", "./Out/page_02.pdf", "./Out/page_03.pdf"] pypdftk.concat(files, "./Out/sample_output.pdf")
def test_concat(self): total_pages = pypdftk.get_num_pages(TEST_PDF_PATH) output_file = pypdftk.concat( [TEST_PDF_PATH, TEST_PDF_PATH, TEST_PDF_PATH]) concat_total_pages = pypdftk.get_num_pages(output_file) self.assertEqual(total_pages * 3, concat_total_pages)
def print_sh4_line( f3x_data, md5_directory, line_number, sh4_list, page_cnt, current_page_num, total_no_of_pages, image_num=None, ): if sh4_list: last_page_cnt = 3 if len(sh4_list) % 3 == 0 else len(sh4_list) % 3 total_fedshare = 0 total_nonfedshare = 0 # total_fednonfed_share = 0 os.makedirs(md5_directory + "SH4/" + line_number, exist_ok=True) sh4_infile = current_app.config["FORM_TEMPLATES_LOCATION"].format( "SH4") for page_num in range(page_cnt): current_page_num += 1 # page_subtotal = 0 memo_array = [] last_page = False schedule_page_dict = {} schedule_page_dict["lineNumber"] = line_number schedule_page_dict["pageNo"] = current_page_num schedule_page_dict["totalPages"] = total_no_of_pages if image_num: schedule_page_dict["IMGNO"] = image_num image_num += 1 page_start_index = page_num * 3 if page_num + 1 == page_cnt: last_page = True # This call prepares data to render on PDF build_sh4_per_page_schedule_dict( last_page, last_page_cnt, page_start_index, schedule_page_dict, sh4_list, memo_array, ) page_fed_subtotal = float(schedule_page_dict["subFedShare"]) page_nonfed_subtotal = float(schedule_page_dict["subNonFedShare"]) schedule_page_dict["subTotalFedNonFedShare"] = "{0:.2f}".format( page_fed_subtotal + page_nonfed_subtotal) total_fedshare += page_fed_subtotal total_nonfedshare += page_nonfed_subtotal if page_cnt == page_num + 1: schedule_page_dict["TotalFedShare"] = "{0:.2f}".format( total_fedshare) schedule_page_dict["totalNonFedShare"] = "{0:.2f}".format( total_nonfedshare) schedule_page_dict["TotalFedNonFedShare"] = "{0:.2f}".format( total_fedshare + total_nonfedshare) schedule_page_dict["committeeName"] = f3x_data["committeeName"] sh4_outfile = (md5_directory + "SH4/" + line_number + "/page_" + str(page_num) + ".pdf") pypdftk.fill_form(sh4_infile, schedule_page_dict, sh4_outfile) # Memo text changes and build memo pages and return updated current_page_num current_page_num, image_num = build_memo_page( memo_array, md5_directory, line_number, current_page_num, page_num, total_no_of_pages, sh4_outfile, name="SH4", image_num=image_num, ) pypdftk.concat( directory_files(md5_directory + "SH4/" + line_number + "/"), md5_directory + "SH4/" + line_number + "/all_pages.pdf", ) if path.isfile(md5_directory + "SH4/all_pages.pdf"): pypdftk.concat( [ md5_directory + "SH4/all_pages.pdf", md5_directory + "SH4/" + line_number + "/all_pages.pdf", ], md5_directory + "SH4/temp_all_pages.pdf", ) os.rename( md5_directory + "SH4/temp_all_pages.pdf", md5_directory + "SH4/all_pages.pdf", ) else: os.rename( md5_directory + "SH4/" + line_number + "/all_pages.pdf", md5_directory + "SH4/all_pages.pdf", ) return image_num
def test_concat(self): total_pages = pypdftk.get_num_pages(TEST_PDF_PATH) output_file = pypdftk.concat([TEST_PDF_PATH, TEST_PDF_PATH, TEST_PDF_PATH]) concat_total_pages = pypdftk.get_num_pages(output_file) self.assertEqual(total_pages * 3, concat_total_pages)
def print_sb_line( f3x_data, md5_directory, line_number, sb_list, page_cnt, current_page_num, total_no_of_pages, image_num=None, ): try: if sb_list: last_page_cnt = 3 if len(sb_list) % 3 == 0 else len(sb_list) % 3 schedule_total = 0 os.makedirs(md5_directory + "SB/" + line_number, exist_ok=True) sb_infile = current_app.config["FORM_TEMPLATES_LOCATION"].format( "SB") for page_num in range(page_cnt): current_page_num += 1 memo_array = [] last_page = False schedule_page_dict = {} schedule_page_dict["lineNumber"] = line_number schedule_page_dict["pageNo"] = current_page_num schedule_page_dict["totalPages"] = total_no_of_pages if image_num: schedule_page_dict["IMGNO"] = image_num image_num += 1 page_start_index = page_num * 3 if page_num + 1 == page_cnt: last_page = True # This call prepares data to render on PDF build_sb_per_page_schedule_dict( last_page, last_page_cnt, page_start_index, schedule_page_dict, sb_list, memo_array, ) try: schedule_total += float(schedule_page_dict["pageSubtotal"]) if page_cnt == page_num + 1: schedule_page_dict["scheduleTotal"] = "{0:.2f}".format( schedule_total) schedule_page_dict["committeeName"] = f3x_data["committeeName"] sb_outfile = (md5_directory + "SB/" + line_number + "/page_" + str(page_num) + ".pdf") pypdftk.fill_form(sb_infile, schedule_page_dict, sb_outfile) # Memo text changes and build memo pages and return updated current_page_num current_page_num, image_num = build_memo_page( memo_array, md5_directory, line_number, current_page_num, page_num, total_no_of_pages, sb_outfile, name="SB", image_num=image_num, ) pypdftk.concat( directory_files(md5_directory + "SB/" + line_number + "/"), md5_directory + "SB/" + line_number + "/all_pages.pdf", ) if path.isfile(md5_directory + "SB/all_pages.pdf"): pypdftk.concat( [ md5_directory + "SB/all_pages.pdf", md5_directory + "SB/" + line_number + "/all_pages.pdf", ], md5_directory + "SB/temp_all_pages.pdf", ) os.rename( md5_directory + "SB/temp_all_pages.pdf", md5_directory + "SB/all_pages.pdf", ) else: os.rename( md5_directory + "SB/" + line_number + "/all_pages.pdf", md5_directory + "SB/all_pages.pdf", ) except: logging.error('**** Start - Error inside if condition ****') # printing stack trace traceback.print_exception(*sys.exc_info()) logging.error('**** End - Error inside if condition ****') return current_page_num, image_num except: # printing stack trace traceback.print_exception(*sys.exc_info())
def make_pdf(self): #Import dependencies from PyPDF2 import PdfFileReader from datetime import datetime import os import pypdftk import pytz pdf_pages = [] #Cycle through pages for j, page in enumerate(self.pages): template_name = os.path.join(THIS_FOLDER, "./pdf_templates/form.pdf") #Read pdf templates using PyPDF2 form = PdfFileReader(open(template_name, "rb")) #Get main form field names from pdf reader fields = form.getFields(tree=None, retval=None, fileobj=None) field_names = list(fields.keys()) #Make a copy of field_values field_values = self.details[:] #Add values from each page for product in page: field_values += [ product.reference, product.lot, product.quantity, product.description ] #Pad out unused fields, zip into dict for writing field_values += [""] * (len(field_names) - len(field_values)) field_dict = dict( zip(field_names, map(lambda x: x.upper(), field_values))) #Add page to writer, update fields from input data pdf_pages.append(pypdftk.fill_form(template_name, field_dict)) if self.checklist: end_form_template_name = os.path.join( THIS_FOLDER, "./pdf_templates/end_page.pdf") #Get pdf templates using PyPDF2 end_form = PdfFileReader(open(end_form_template_name, "rb")) #Get end form fields from reader end_fields = end_form.getFields(tree=None, retval=None, fileobj=None) end_field_names = list(end_fields.keys()) #Populate end field values with name and date, position depending on options end_field_values = [""] * 4 index = 2 if self.new else 0 tz = pytz.timezone("Australia/Brisbane") current_date = datetime.now(tz) end_field_values[index:index + 1] = [ self.client.first_name + " " + self.client.last_name, current_date.strftime("%d/%m/%Y") ] #Zip end field values and names into dict end_field_dict = dict(zip(end_field_names, end_field_values)) pdf_pages.append( pypdftk.fill_form(end_form_template_name, end_field_dict)) pypdftk.concat(pdf_pages, os.path.join(THIS_FOLDER, "../dynamic/print.pdf"))
def print_pdftk(stamp_print): # check if json_file is in the request # try: if 'json_file' in request.files: total_no_of_pages = 0 page_no = 1 has_sa_schedules = has_sb_schedules = False json_file = request.files.get('json_file') # generate md5 for json file # FIXME: check if PDF already exist with md5, if exist return pdf instead of re-generating PDF file. json_file_md5 = utils.md5_for_file(json_file) json_file.stream.seek(0) md5_directory = current_app.config['OUTPUT_DIR_LOCATION'].format(json_file_md5) os.makedirs(md5_directory, exist_ok=True) infile = current_app.config['FORM_TEMPLATES_LOCATION'].format('F3L') # save json file as md5 file name json_file.save(current_app.config['REQUEST_FILE_LOCATION'].format(json_file_md5)) outfile = md5_directory + json_file_md5 + '_temp.pdf' # load json file f3l_json = json.load(open(current_app.config['REQUEST_FILE_LOCATION'].format(json_file_md5))) # setting timestamp and imgno to empty as these needs to show up after submission output = {} if stamp_print != 'stamp': output['FILING_TIMESTAMP'] = '' output['IMGNO'] = '' # read data from json file f3l_data = f3l_json['data'] # check if summary is present in fecDataFile f3l_summary = [] if 'summary' in f3l_data: f3l_summary = f3l_data['summary'] # split coverage start date and coverage end date to set month, day, and year if f3l_data['coverageStartDate'] and len(f3l_data['coverageStartDate']) > 0: coverage_start_date_array = f3l_data['coverageStartDate'].split("/") f3l_data['coverageStartDateMonth'] = coverage_start_date_array[0] f3l_data['coverageStartDateDay'] = coverage_start_date_array[1] f3l_data['coverageStartDateYear'] = coverage_start_date_array[2] if f3l_data['coverageEndDate'] and len(f3l_data['coverageEndDate']) > 0: coverage_end_date_array = f3l_data['coverageEndDate'].split("/") f3l_data['coverageEndDateMonth'] = coverage_end_date_array[0] f3l_data['coverageEndDateDay'] = coverage_end_date_array[1] f3l_data['coverageEndDateYear'] = coverage_end_date_array[2] # checking for signed date, it is only available for submitted reports if f3l_data['date_signed'] and len(f3l_data['date_signed']) > 0: date_signed_array = f3l_data['date_signed'].split("-") f3l_data['dateSignedMonth'] = date_signed_array[0] f3l_data['dateSignedDay'] = date_signed_array[1] f3l_data['dateSignedYear'] = date_signed_array[2] # build treasurer name to map it to PDF template treasurer_full_name = [] treasurer_full_name.append(f3l_data['treasurerLastName']) treasurer_full_name.append(f3l_data['treasurerFirstName']) treasurer_full_name.append(f3l_data['treasurerMiddleName']) treasurer_full_name.append(f3l_data['treasurerPrefix']) treasurer_full_name.append(f3l_data['treasurerSuffix']) f3l_data['treasurerFullName'] = ",".join(map(str, treasurer_full_name)) f3l_data['treasurerName'] = f3l_data['treasurerLastName'] + "," + f3l_data['treasurerFirstName'] f3l_data['efStamp'] = '[Electronically Filed]' # checking if json contains summary details, for individual transactions print there wouldn't be summary if len(f3l_summary) > 0: total_no_of_pages = 1 f3l_data_summary_array = [f3l_data, f3l_summary] if 'memoText' in f3l_data and f3l_data['memoText']: total_no_of_pages += 1 else: f3l_data_summary_array = [f3l_data] f3l_data_summary = {i: j for x in f3l_data_summary_array for i, j in x.items()} # process all schedules and build the PDF's process_output, total_no_of_pages = process_schedules(f3l_data, md5_directory, total_no_of_pages) has_sa_schedules = process_output.get('has_sa_schedules') has_sb_schedules = process_output.get('has_sb_schedules') if len(f3l_summary) > 0: get_summary_detail(f3l_summary, f3l_data, f3l_data_summary) f3l_data_summary['PAGESTR'] = "PAGE " + str(page_no) + " / " + str(total_no_of_pages) pypdftk.fill_form(infile, f3l_data_summary, outfile) shutil.copy(outfile, md5_directory + 'F3L_Summary.pdf') os.remove(md5_directory + json_file_md5 + '_temp.pdf') # Memo text changes if 'memoText' in f3l_data_summary and f3l_data_summary['memoText']: memo_dict = {} temp_memo_outfile = md5_directory + 'F3L_Summary_memo.pdf' memo_infile = current_app.config['FORM_TEMPLATES_LOCATION'].format('TEXT') memo_dict['scheduleName_1'] = 'F3L' + f3l_data_summary['amendmentIndicator'] memo_dict['memoDescription_1'] = f3l_data_summary['memoText'] memo_dict['PAGESTR'] = "PAGE " + str(2) + " / " + str(total_no_of_pages) pypdftk.fill_form(memo_infile, memo_dict, temp_memo_outfile) pypdftk.concat([md5_directory + 'F3L_Summary.pdf', temp_memo_outfile], md5_directory + json_file_md5 + '_temp.pdf') shutil.copy(md5_directory + json_file_md5 + '_temp.pdf', md5_directory + 'F3L_Summary.pdf') os.remove(md5_directory + json_file_md5 + '_temp.pdf') # check if all_pages already exsits if os.path.exists(md5_directory + 'all_pages.pdf'): os.remove(md5_directory + 'all_pages.pdf') # checking for sa transactions if has_sa_schedules: pypdftk.concat([md5_directory + 'F3L_Summary.pdf', md5_directory + 'SA/all_pages.pdf'], md5_directory + 'all_pages.pdf') os.remove(md5_directory + 'SA/all_pages.pdf') shutil.rmtree(md5_directory + 'SA') else: shutil.copy(md5_directory + 'F3L_Summary.pdf', md5_directory + 'all_pages.pdf') # checking for sb transactions if has_sb_schedules: pypdftk.concat([md5_directory + 'all_pages.pdf', md5_directory + 'SB/all_pages.pdf'], md5_directory + 'temp_all_pages.pdf') shutil.move(md5_directory + 'temp_all_pages.pdf', md5_directory + 'all_pages.pdf') os.remove(md5_directory + 'SB/all_pages.pdf') shutil.rmtree(md5_directory + 'SB') else: # no summary, expecting it to be from individual transactions if has_sa_schedules: if os.path.exists(md5_directory + 'all_pages.pdf'): os.remove(md5_directory + 'all_pages.pdf') shutil.move(md5_directory + 'SA/all_pages.pdf', md5_directory + 'all_pages.pdf') else: shutil.move(md5_directory + 'SA/all_pages.pdf', md5_directory + 'all_pages.pdf') shutil.rmtree(md5_directory + 'SA') if has_sb_schedules: if os.path.exists(md5_directory + 'all_pages.pdf'): os.remove(md5_directory + 'all_pages.pdf') shutil.move(md5_directory + 'SB/all_pages.pdf', md5_directory + 'all_pages.pdf') else: shutil.move(md5_directory + 'SB/all_pages.pdf', md5_directory + 'all_pages.pdf') shutil.rmtree(md5_directory + 'SB') # push output file to AWS s3 = boto3.client('s3') s3.upload_file(md5_directory + 'all_pages.pdf', current_app.config['AWS_FECFILE_COMPONENTS_BUCKET_NAME'], md5_directory + 'all_pages.pdf', ExtraArgs={'ContentType': "application/pdf", 'ACL': "public-read"}) response = { # 'file_name': '{}.pdf'.format(json_file_md5), 'pdf_url': current_app.config['PRINT_OUTPUT_FILE_URL'].format(json_file_md5) + 'all_pages.pdf' } # return response if flask.request.method == "POST": envelope = common.get_return_envelope( data=response ) status_code = status.HTTP_201_CREATED return flask.jsonify(**envelope), status_code else: if flask.request.method == "POST": envelope = common.get_return_envelope( 'false', 'JSON file is missing from your request' ) status_code = status.HTTP_400_BAD_REQUEST return flask.jsonify(**envelope), status_code
def process_sa_line(f3l_data, md5_directory, line_number, sa_line, sa_line_page_cnt, sa_line_start_page, sa_line_last_page_cnt, total_no_of_pages): has_sa_schedules = False if len(sa_line) > 0: sa_line_start_page += 1 has_sa_schedules = True schedule_total = 0.00 schedule_aggregate_total = 0.00 os.makedirs(md5_directory + 'SA/' + line_number, exist_ok=True) sa_infile = current_app.config['FORM_TEMPLATES_LOCATION'].format('SA3L') if sa_line_page_cnt > 0: for sa_page_no in range(sa_line_page_cnt): # if sa_schedule_page_dict['memoDescription']: # sa_memo_obj = {'memoDescripton':sa_schedule_page_dict['memoDescription'], 'transactionId': sa_schedule_page_dict['transactionId']} # sa_memo.append(sa_memo_obj) page_subtotal = 0.00 memo_array = [] last_page = False sa_schedule_page_dict = {} sa_schedule_page_dict['lineNumber'] = line_number sa_schedule_page_dict['pageNo'] = sa_line_start_page + sa_page_no sa_schedule_page_dict['totalPages'] = total_no_of_pages page_start_index = sa_page_no * 4 if ((sa_page_no + 1) == sa_line_page_cnt): last_page = True # This call prepares data to render on PDF sa_schedule_dict = build_sa_per_page_schedule_dict(last_page, sa_line_last_page_cnt, page_start_index, sa_schedule_page_dict, sa_line, memo_array) page_subtotal = float(sa_schedule_page_dict['pageSubtotal']) page_aggregate_total = float(sa_schedule_page_dict['pageAggSubtotal']) schedule_total += page_subtotal schedule_aggregate_total += page_aggregate_total if sa_line_page_cnt == (sa_page_no + 1): sa_schedule_page_dict['scheduleTotal'] = '{0:.2f}'.format(schedule_total) sa_schedule_page_dict['scheduleAggTotal'] = '{0:.2f}'.format(page_aggregate_total) sa_schedule_page_dict['committeeName'] = f3l_data['committeeName'] sa_outfile = md5_directory + 'SA/' + line_number + '/page_' + str(sa_page_no) + '.pdf' pypdftk.fill_form(sa_infile, sa_schedule_page_dict, sa_outfile) # Memo text changes memo_dict = {} if len(memo_array) >= 1: temp_memo_outfile = md5_directory + 'SA/' + line_number + '/page_memo_temp.pdf' memo_infile = current_app.config['FORM_TEMPLATES_LOCATION'].format('TEXT') memo_outfile = md5_directory + 'SA/' + line_number + '/page_memo_' + str(sa_page_no) + '.pdf' memo_dict['scheduleName_1'] = memo_array[0]['scheduleName'] memo_dict['memoDescription_1'] = memo_array[0]['memoDescription'] memo_dict['transactionId_1'] = memo_array[0]['transactionId'] if len(memo_array) >= 2: memo_dict['scheduleName_2'] = memo_array[1]['scheduleName'] memo_dict['memoDescription_2'] = memo_array[1]['memoDescription'] memo_dict['transactionId_2'] = memo_array[1]['transactionId'] # build page pypdftk.fill_form(memo_infile, memo_dict, memo_outfile) pypdftk.concat([sa_outfile, memo_outfile], temp_memo_outfile) os.remove(memo_outfile) os.rename(temp_memo_outfile, sa_outfile) sa_line_start_page += 1 if len(memo_array) >= 3: memo_dict = {} memo_outfile = md5_directory + 'SA/' + line_number + '/page_memo_' + str(sa_page_no) + '.pdf' memo_dict['scheduleName_1'] = memo_array[2]['scheduleName'] memo_dict['memoDescription_1'] = memo_array[2]['memoDescription'] memo_dict['transactionId_1'] = memo_array[2]['transactionId'] if len(memo_array) >= 4: memo_dict['scheduleName_2'] = memo_array[3]['scheduleName'] memo_dict['memoDescription_2'] = memo_array[3]['memoDescription'] memo_dict['transactionId_2'] = memo_array[3]['transactionId'] pypdftk.fill_form(memo_infile, memo_dict, memo_outfile) pypdftk.concat([sa_outfile, memo_outfile], temp_memo_outfile) os.remove(memo_outfile) os.rename(temp_memo_outfile, sa_outfile) sa_line_start_page += 1 pypdftk.concat(directory_files(md5_directory + 'SA/' + line_number + '/'), md5_directory + 'SA/' + line_number + '/all_pages.pdf') # if len(sa_memo) > 0: # for sa_memo_obj in range(sa_memo): # sa_outfile = md5_directory + 'SA/' + line_number + '/page_' + str(sa_page_no) + '.pdf' # pypdftk.fill_form(sa_infile, sa_schedule_page_dict, sa_outfile) # if all_pages.pdf exists in SA folder, concatenate line number pdf to all_pages.pdf if path.isfile(md5_directory + 'SA/all_pages.pdf'): pypdftk.concat([md5_directory + 'SA/all_pages.pdf', md5_directory + 'SA/' + line_number + '/all_pages.pdf'], md5_directory + 'SA/temp_all_pages.pdf') os.rename(md5_directory + 'SA/temp_all_pages.pdf', md5_directory + 'SA/all_pages.pdf') else: os.rename(md5_directory + 'SA/' + line_number + '/all_pages.pdf', md5_directory + 'SA/all_pages.pdf') return has_sa_schedules