def error(msg): if flask.request.method == "POST": envelope = common.get_return_envelope( 'false', msg ) status_code = status.HTTP_400_BAD_REQUEST return flask.jsonify(**envelope), status_code
def paginate(file_content=None, begin_image_num=None): if file_content and begin_image_num: # if "json_file_name" in request.json: # # json_file_name = request.json.get("json_file_name") # txn_img_num = request.json.get("begin_image_num") # if not txn_img_num: # if flask.request.method == "POST": # envelope = common.get_return_envelope( # "false", "begin_image_num is missing from your request" # ) # status_code = status.HTTP_400_BAD_REQUEST # return flask.jsonify(**envelope), status_code # file_url = current_app.config["AWS_S3_FECFILE_COMPONENTS_DOMAIN"] + "/" + json_file_name + ".json" # file_url = "https://dev-efile-repo.s3.amazonaws.com/" + json_file_name + ".json" # with urllib.request.urlopen(file_url) as url: # file_content = url.read().decode() f1m_json = json.loads(file_content) data = f1m_json["data"] txn_img_json = { "summary": { "committeeId": data.get("committeeId", None), "begin_image_num": begin_image_num, "end_image_num": begin_image_num } } total_no_of_pages = 1 # return True, {"total_pages": total_no_of_pages, "txn_img_json": txn_img_json} response = { "total_pages": total_no_of_pages, "txn_img_json": txn_img_json } # if flask.request.method == "POST": envelope = common.get_return_envelope(data=response) status_code = status.HTTP_200_OK return flask.jsonify(**envelope), status_code else: # if flask.request.method == "POST": envelope = common.get_return_envelope( False, "json_file_name is missing from your request") return flask.jsonify(**envelope), status.HTTP_400_BAD_REQUEST
def handle_general_exception(error): """ Flask Error Handler for Exception :param str error: error message :return: tuple(return envelope string, http status code int) :rtype: tuple(str, int) """ envelope = common.get_return_envelope(success="false", message=str(error)) LOGGER.exception(str(error)) return flask.jsonify(**envelope), 500
def print_pdftk(stamp_print): # check if json_file is in the request if 'json_file' in request.files: total_no_of_pages = 5 page_no = 1 json_file = request.files.get('json_file') # generate md5 for json file json_file_md5 = utils.md5_for_file(json_file) json_file.stream.seek(0) md5_directory = current_app.config['OUTPUT_DIR_LOCATION'].format( json_file_md5) os.makedirs(md5_directory, exist_ok=True) infile = current_app.config['FORM_TEMPLATES_LOCATION'].format('F3X') # save json file as md5 file name json_file.save( current_app.config['REQUEST_FILE_LOCATION'].format(json_file_md5)) outfile = md5_directory + json_file_md5 + '_temp.pdf' f3x_json = json.load( open(current_app.config['REQUEST_FILE_LOCATION'].format( json_file_md5))) # setting timestamp and imgno to empty as these needs to show up after submission if stamp_print != 'stamp': f3x_json['FILING_TIMESTAMP'] = '' f3x_json['IMGNO'] = '' # f3x_json = json.loads(json_data) f3x_data = f3x_json['data'] f3x_summary_temp = f3x_data['summary'] f3x_summary = {'cashOnHandYear': f3x_summary_temp['cashOnHandYear']} f3x_col_a = f3x_summary_temp['colA'] f3x_col_b = f3x_summary_temp['colB'] for key in f3x_col_a: f3x_summary['colA_' + key] = f3x_col_a[key] for key in f3x_col_b: f3x_summary['colB_' + key] = f3x_col_b[key] f3x_data_summary_array = [f3x_data, f3x_summary] f3x_data_summary = { i: j for x in f3x_data_summary_array for i, j in x.items() } if 'schedules' in f3x_data: schedules = f3x_data['schedules'] if 'SA' in schedules: schedule_total = 0.00 # os.remove(md5_directory + 'SA/all_pages.pdf') # create SA folder under MD5 directory os.makedirs(md5_directory + 'SA', exist_ok=True) sa_infile = current_app.config[ 'FORM_TEMPLATES_LOCATION'].format('SA') sa_schedules = schedules['SA'] # print(len(sa_schedules)) sa_array = [] sa_json = {} no_of_pages = 0 no_of_transactions_in_last_page = 0 # print(int(len(sa_schedules) / 3)) if int(len(sa_schedules) % 3) == 0: no_of_pages = int(len(sa_schedules) / 3) no_of_transactions_in_last_page = 3 else: no_of_pages = int(len(sa_schedules) / 3) + 1 no_of_transactions_in_last_page = int( len(sa_schedules) % 3) total_no_of_pages += no_of_pages if no_of_pages > 1: for sa_page_no in range(no_of_pages): page_subtotal = 0.00 sa_schedule_page_dict = {} sa_schedule_page_dict['pageNo'] = 5 + sa_page_no + 1 sa_schedule_page_dict['totalPages'] = total_no_of_pages page_start_index = sa_page_no * 3 if sa_page_no == (no_of_pages - 1): # page_end_index = page_start_index + no_of_transactions_in_last_page - 1 sa_schedule_dict = build_per_page_schedule_dict( no_of_transactions_in_last_page, page_start_index, sa_schedule_page_dict, sa_schedules) else: # no_of_transactions_in_last_page = 3 sa_schedule_dict = build_per_page_schedule_dict( 3, page_start_index, sa_schedule_page_dict, sa_schedules) page_subtotal = float( sa_schedule_page_dict['pageSubtotal']) schedule_total += page_subtotal if no_of_pages == (sa_page_no + 1): sa_schedule_page_dict[ 'scheduleTotal'] = '{0:.2f}'.format( schedule_total) sa_schedule_page_dict['committeeName'] = f3x_data[ 'committeeName'] sa_schedule_page_dict['lineNumber'] = sa_schedule_dict[ 'lineNumber'] sa_outfile = md5_directory + '/SA/' + 'page_' + str( sa_page_no) + '.pdf' pypdftk.fill_form(sa_infile, sa_schedule_page_dict, sa_outfile) pypdftk.concat(directory_files(md5_directory + 'SA/'), md5_directory + 'SA/all_pages.pdf') f3x_data_summary['PAGESTR'] = "PAGE " + str(page_no) + " / " + str( total_no_of_pages) pypdftk.fill_form(infile, f3x_data_summary, outfile) shutil.copy(outfile, md5_directory + 'F3X.pdf') os.remove(md5_directory + json_file_md5 + '_temp.pdf') # pypdftk.concat(directory_files(md5_directory + 'SA/'), md5_directory + 'SA/all_pages.pdf') pypdftk.concat( [md5_directory + 'F3X.pdf', md5_directory + 'SA/all_pages.pdf'], md5_directory + 'all_pages.pdf') os.remove(md5_directory + 'SA/all_pages.pdf') # push output file to AWS s3 = boto3.client('s3') s3.upload_file( md5_directory + 'all_pages.pdf', current_app.config['AWS_FECFILE_COMPONENTS_BUCKET_NAME'], md5_directory + 'all_pages.pdf', ExtraArgs={ 'ContentType': "application/pdf", 'ACL': "public-read" }) response = { # 'file_name': '{}.pdf'.format(json_file_md5), 'pdf_url': current_app.config['PRINT_OUTPUT_FILE_URL'].format(json_file_md5) + 'all_pages.pdf' } if flask.request.method == "POST": envelope = common.get_return_envelope(data=response) status_code = status.HTTP_201_CREATED return flask.jsonify(**envelope), status_code else: if flask.request.method == "POST": envelope = common.get_return_envelope( 'false', 'JSON file is missing from your request') status_code = status.HTTP_400_BAD_REQUEST return flask.jsonify(**envelope), status_code
def print_f99_pdftk_html( stamp_print="", paginate=False, begin_image_num=None, page_count=False, file_content=None, silent_print=False, filing_timestamp=None, rep_id=None, attachment_file_content=None, ): # check if json_file is in the request # HTML("templates/forms/test.html").write_pdf("output/pdf/test/test.pdf") # HTML(string='''<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><div><b>This is bold text</b></div><div><u>This is underline text</u></div><div><i>This is italics text</i><u><br></u></div><div align='center'><u>Title</u></div><div align='left'><u><br></u></div><ol><li>one</li><li>two</li><li>three</li></ol>''').write_pdf("output/pdf/test/test.pdf") # pdfkit.from_file("templates/forms/test.html", "output/pdf/test/test.pdf") # pypdftk.stamp(current_app.config['FORM_TEMPLATES_LOCATION'].format('F99'), "output/pdf/test/test.pdf", "output/pdf/test/output.pdf") try: silent_print = silent_print txn_img_num = begin_image_num filing_timestamp = filing_timestamp if ((page_count and file_content) or ((paginate or silent_print) and file_content and begin_image_num) or (not paginate and "json_file" in request.files)): if page_count and file_content: json_file_md5 = md5_for_text(file_content) json_data = json.loads(file_content) elif (paginate or silent_print) and file_content and begin_image_num: # generate md5 for file_content json_file_md5 = md5_for_text(file_content) json_data = json.loads(file_content) elif not paginate and "json_file" in request.files: json_file = request.files.get("json_file") silent_print = (True if request.form.get("silent_print") and request.form.get("silent_print").lower() in ["true", "1"] else False) page_count = (True if request.form.get("page_count") and request.form.get("page_count").lower() in ["true", "1"] else False) if silent_print: txn_img_num = request.form.get("begin_image_num", None) if not txn_img_num: if flask.request.method == "POST": envelope = common.get_return_envelope( "false", "begin_image_num is missing from your request") status_code = status.HTTP_400_BAD_REQUEST return flask.jsonify(**envelope), status_code txn_img_num = int(txn_img_num) filing_timestamp = request.form.get( "filing_timestamp", None) json_file_md5 = md5_for_file(json_file) json_file.stream.seek(0) # save json file as md5 file name json_file.save( current_app.config["REQUEST_FILE_LOCATION"].format( json_file_md5)) # load json file json_data = json.load( open(current_app.config["REQUEST_FILE_LOCATION"].format( json_file_md5))) md5_directory = current_app.config["OUTPUT_DIR_LOCATION"].format( json_file_md5) # if paginate or page_count is True and directory exist then don't remove it is_dir_exist = False if os.path.isdir(md5_directory): is_dir_exist = True os.makedirs(md5_directory, exist_ok=True) # os.makedirs(md5_directory + "images", exist_ok=True) if not os.path.exists(md5_directory + "images"): shutil.copytree("templates/forms/F99/images", md5_directory + "images") shutil.copyfile("templates/forms/F99/form-text.css", md5_directory + "form-text.css") infile = current_app.config["HTML_FORM_TEMPLATES_LOCATION"].format( "template") outfile = md5_directory + json_file_md5 + ".html" form99_json_data = json_data["data"] with open(infile) as inf: txt = inf.read() soup = bs4.BeautifulSoup(txt, features="html5lib") soup.find("label", attrs={ "id": "committeeName" }).string = form99_json_data["committeeName"] soup.find("label", attrs={ "id": "street1" }).string = form99_json_data["street1"] soup.find("label", attrs={ "id": "street2" }).string = form99_json_data["street2"] soup.find("label", attrs={ "id": "city" }).string = form99_json_data["city"] soup.find("label", attrs={ "id": "state" }).string = form99_json_data["state"] soup.find("label", attrs={ "id": "zipCode" }).string = form99_json_data["zipCode"] soup.find("span", attrs={ "id": "committeeId" }).string = form99_json_data["committeeId"] name_list = [ "LastName", "FirstName", "MiddleName", "Prefix", "Suffix" ] treasurerFullName = "" for item in name_list: item = "treasurer" + item if form99_json_data.get(item): treasurerFullName += form99_json_data.get(item) + ", " soup.find("label", attrs={ "id": "treasurerFullName" }).string = treasurerFullName[:-2] soup.find("label", attrs={ "id": "treasurerName" }).string = ((form99_json_data.get("treasurerLastName", "") + ", " + form99_json_data.get("treasurerFirstName", "") ).strip().rstrip(",").strip()) f99_html_data = form99_json_data["text"] soup.find("label", attrs={"id": "text"}).string = f99_html_data soup.find("label", attrs={ "id": form99_json_data["reason"] }).string = "X" date_array = form99_json_data["dateSigned"].split("/") soup.find("span", attrs={ "id": "dateSignedMonth" }).string = str(date_array[0]) soup.find("span", attrs={ "id": "dateSignedDate" }).string = str(date_array[1]) soup.find("span", attrs={ "id": "dateSignedYear" }).string = str(date_array[2]) with open(outfile, "w") as output_file: output_file.write( str(soup).replace("<", "<").replace(">", ">")) # F99 PDF page padding options options = { "margin-top": "0.40in", "margin-right": "0.20in", "margin-bottom": "0.40in", "margin-left": "0.20in", } # HTML(outfile).write_pdf(md5_directory + json_file_md5 + '.pdf', stylesheets=[CSS(current_app.config['FORMS_LOCATION'].format('F99.css'))]) pdfkit.from_file(outfile, md5_directory + json_file_md5 + ".pdf", options=options) # pdfkit.from_file(outfile, md5_directory + json_file_md5 + '.pdf') total_no_of_pages = pypdftk.get_num_pages(md5_directory + json_file_md5 + ".pdf") # checking if attachment_file exist if ((paginate or page_count) and attachment_file_content) or ( not paginate and "attachment_file" in request.files): # reading Attachment title file attachment_title_file = current_app.config[ "FORM_TEMPLATES_LOCATION"].format("Attachment_Title") if (paginate or page_count) and attachment_file_content: attachment_file = json.loads(attachment_file_content) else: attachment_file = request.files.get("attachment_file") attachment_file.save( os.path.join(md5_directory + "attachment_temp.pdf")) os.makedirs(md5_directory + "attachment", exist_ok=True) os.makedirs(md5_directory + "final_attachment", exist_ok=True) pypdftk.split(md5_directory + "attachment_temp.pdf", md5_directory + "attachment") os.remove(md5_directory + "attachment/doc_data.txt") attachment_no_of_pages = pypdftk.get_num_pages( os.path.join(md5_directory + "attachment_temp.pdf")) attachment_page_no = total_no_of_pages total_no_of_pages += attachment_no_of_pages # we are doing this to assign page numbers to attachment file for filename in os.listdir(md5_directory + "attachment"): attachment_page_no += 1 page_dict = {} page_dict["PAGESTR"] = ("PAGE " + str(attachment_page_no) + " / " + str(total_no_of_pages)) if silent_print: page_dict["IMGNO"] = txn_img_num + attachment_page_no pypdftk.fill_form( attachment_title_file, md5_directory + "attachment/attachment_page_" + str(attachment_page_no) + ".pdf", ) pypdftk.stamp( md5_directory + "attachment/" + filename, md5_directory + "attachment/attachment_page_" + str(attachment_page_no) + ".pdf", md5_directory + "final_attachment/attachment_" + str(attachment_page_no) + ".pdf", ) pypdftk.concat( directory_files(md5_directory + "final_attachment/"), md5_directory + "attachment.pdf", ) os.remove(md5_directory + "attachment_temp.pdf") os.makedirs(md5_directory + "pages", exist_ok=True) os.makedirs(md5_directory + "final_pages", exist_ok=True) pypdftk.split(md5_directory + json_file_md5 + ".pdf", md5_directory + "pages") os.remove(md5_directory + "pages/doc_data.txt") f99_page_no = 1 for filename in os.listdir(md5_directory + "pages"): page_dict = {} page_dict["PAGESTR"] = ("PAGE " + str(f99_page_no) + " / " + str(total_no_of_pages)) if silent_print: page_dict["IMGNO"] = txn_img_num txn_img_num += 1 # need to print timestamp on first page only if filing_timestamp and f99_page_no == 1: page_dict["FILING_TIMESTAMP"] = filing_timestamp page_number_file = current_app.config[ "FORM_TEMPLATES_LOCATION"].format("Page_Number") pypdftk.fill_form( page_number_file, page_dict, md5_directory + "pages/page_number_" + str(f99_page_no).zfill(6) + ".pdf", ) pypdftk.stamp( md5_directory + "pages/page_number_" + str(f99_page_no).zfill(6) + ".pdf", md5_directory + "pages/" + filename, md5_directory + "final_pages/page_" + str(f99_page_no).zfill(6) + ".pdf", ) f99_page_no += 1 pypdftk.concat( directory_files(md5_directory + "final_pages/"), json_file_md5 + "_temp.pdf", ) if ((paginate or page_count) and attachment_file_content) or ( not paginate and "attachment_file" in request.files): pypdftk.concat( [ json_file_md5 + "_temp.pdf", md5_directory + "attachment.pdf" ], md5_directory + "all_pages.pdf", ) shutil.rmtree(md5_directory + "attachment") shutil.rmtree(md5_directory + "final_attachment") os.remove(md5_directory + "attachment.pdf") else: shutil.move(json_file_md5 + "_temp.pdf", md5_directory + "all_pages.pdf") # clean up task shutil.rmtree(md5_directory + "pages") shutil.rmtree(md5_directory + "final_pages") os.remove(md5_directory + json_file_md5 + ".pdf") # if flask.request.method == "POST": response = { # 'file_name': ent_app.conf'{}.pdf'.format(json_file_md5), "total_pages": total_no_of_pages, } if not page_count and not paginate: s3 = boto3.client("s3") extraArgs = { "ContentType": "application/pdf", "ACL": "public-read" } if silent_print: response["pdf_url"] = current_app.config[ 'S3_FILE_URL'] + rep_id + '.pdf' s3.upload_file( md5_directory + 'all_pages.pdf', current_app. config['AWS_FECFILE_COMPONENTS_BUCKET_NAME'], current_app.config['AWS_FECFILE_OUTPUT_DIRECTORY'] + '/' + str(rep_id) + '.pdf', ExtraArgs=extraArgs) else: response["pdf_url"] = ( current_app.config["PRINT_OUTPUT_FILE_URL"].format( json_file_md5) + "all_pages.pdf", ) s3.upload_file( md5_directory + "all_pages.pdf", current_app. config["AWS_FECFILE_COMPONENTS_BUCKET_NAME"], md5_directory + "all_pages.pdf", ExtraArgs=extraArgs, ) else: if not is_dir_exist: shutil.rmtree(md5_directory) if paginate: txn_img_json = { "summary": { "committeeId": form99_json_data.get("committeeId", None), "begin_image_num": begin_image_num, "end_image_num": txn_img_num } } response["txn_img_json"] = txn_img_json envelope = common.get_return_envelope(data=response) status_code = (status.HTTP_200_OK if page_count or paginate else status.HTTP_201_CREATED) return flask.jsonify(**envelope), status_code # elif page_count or paginate: # if not is_dir_exist: # shutil.rmtree(md5_directory) # response = { # "total_pages": total_no_of_pages, # } # elif paginate: # txn_img_json = { # 'summary' : { # 'committeeId': form99_json_data.get('committeeId', None) # } # } # response['txn_img_json'] = txn_img_json # return True, response # elif silent_print and not flask.request.method == "POST": # return True, {} else: if paginate or page_count or silent_print: envelope = common.get_return_envelope(False, "") else: # elif flask.request.method == "POST": envelope = common.get_return_envelope( False, "json_file is missing from your request") return flask.jsonify(**envelope), status.HTTP_400_BAD_REQUEST except Exception as e: traceback.print_exception(*sys.exc_info()) return error("Error generating print preview, error message: " + str(e))
def print_pdftk( stamp_print="", page_count=False, file_content=None, begin_image_num=None, silent_print=False, filing_timestamp=None, rep_id=None, ): # check if json_file_name is in the request try: silent_print = silent_print txn_img_num = begin_image_num filing_timestamp = filing_timestamp if "json_file" in request.files or (page_count and file_content): if "json_file" in request.files: json_file = request.files.get("json_file") silent_print = (True if request.form.get("silent_print") and request.form.get("silent_print").lower() in ["true", "1"] else False) page_count = (True if request.form.get("page_count") and request.form.get("page_count").lower() in ["true", "1"] else False) if silent_print: txn_img_num = request.form.get("begin_image_num", None) if not txn_img_num: if flask.request.method == "POST": envelope = common.get_return_envelope( "false", "begin_image_num is missing from your request") status_code = status.HTTP_400_BAD_REQUEST return flask.jsonify(**envelope), status_code txn_img_num = int(txn_img_num) filing_timestamp = request.form.get( "filing_timestamp", None) json_file_md5 = md5_for_file(json_file) json_file.stream.seek(0) # save json file as md5 file name json_file.save( current_app.config["REQUEST_FILE_LOCATION"].format( json_file_md5)) # load json file f1m_json = json.load( open(current_app.config["REQUEST_FILE_LOCATION"].format( json_file_md5))) # if page_count is True then return from here elif page_count and file_content: response = {"total_pages": 1} # if flask.request.method == "POST": envelope = common.get_return_envelope(data=response) return flask.jsonify(**envelope), status.HTTP_200_OK elif silent_print and begin_image_num and file_content: json_file_md5 = md5_for_text(file_content) f1m_json = json.loads(file_content) md5_directory = current_app.config["OUTPUT_DIR_LOCATION"].format( json_file_md5) # deleting directory if it exists and has any content delete_directory(md5_directory) os.makedirs(md5_directory, exist_ok=True) infile = current_app.config["FORM_TEMPLATES_LOCATION"].format( "F1M") outfile = md5_directory + json_file_md5 + "_temp.pdf" # setting timestamp and imgno to empty as these needs to show up after submission if stamp_print != "stamp": f1m_json["FILING_TIMESTAMP"] = "" f1m_json["IMGNO"] = "" # read data from json file f1m_data = f1m_json["data"] # adding txn_img_num if silent_print is True if silent_print: f1m_data["IMGNO"] = txn_img_num if filing_timestamp: f1m_data["FILING_TIMESTAMP"] = filing_timestamp name_list = [ "LastName", "FirstName", "MiddleName", "Prefix", "Suffix" ] # build treasurer name to map it to PDF template treasurerFullName = "" for item in name_list: item = "treasurer" + item if f1m_data.get(item): treasurerFullName += f1m_data.get(item) + " " f1m_data["treasurerFullName"] = treasurerFullName[:-1] f1m_data["treasurerName"] = ( f1m_data.get("treasurerLastName", "") + ", " + f1m_data.get("treasurerFirstName", "")) f1m_data["treasurerName"] = ( f1m_data["treasurerName"].strip().rstrip(",").strip()) f1m_data["efStamp"] = "[Electronically Filed]" if "candidates" in f1m_data: for candidate in f1m_data["candidates"]: candidateFullName = "" for item in name_list: item = "candidate" + item if f1m_data.get(item): candidateFullName += f1m_data.get(item) + " " f1m_data["candidateName" + str( candidate["candidateNumber"])] = candidateFullName[:-1] f1m_data["candidateOffice" + str(candidate["candidateNumber"] )] = candidate["candidateOffice"] f1m_data["candidateStateDist" + str(candidate["candidateNumber"])] = "/ ".join( map( str, [ candidate["candidateState"], candidate["candidateDistrict"], ], )) f1m_data["contributionDate" + str(candidate["candidateNumber"] )] = candidate["contributionDate"] os.makedirs(md5_directory + str(f1m_data["reportId"]) + "/", exist_ok=True) infile = current_app.config["FORM_TEMPLATES_LOCATION"].format( "F1M") pypdftk.fill_form(infile, f1m_data, outfile) shutil.copy(outfile, md5_directory + str(f1m_data["reportId"]) + "/F1M.pdf") os.remove(outfile) # 'file_name': '{}.pdf'.format(json_file_md5), response = {"total_pages": 1} if not page_count: s3 = boto3.client("s3") extraArgs = { "ContentType": "application/pdf", "ACL": "public-read" } if silent_print: response["pdf_url"] = ( current_app. config["AWS_FECFILE_COMPONENTS_BUCKET_NAME"], rep_id + ".pdf", ) s3.upload_file( md5_directory + str(f1m_data["reportId"]) + "/F1M.pdf", current_app. config["AWS_FECFILE_COMPONENTS_BUCKET_NAME"], rep_id + ".pdf", ExtraArgs=extraArgs, ) else: response["pdf_url"] = ( current_app.config["PRINT_OUTPUT_FILE_URL"].format( json_file_md5) + "F1M.pdf", ) s3.upload_file( md5_directory + str(f1m_data["reportId"]) + "/F1M.pdf", current_app. config["AWS_FECFILE_COMPONENTS_BUCKET_NAME"], md5_directory + "F1M.pdf", ExtraArgs=extraArgs, ) # return response # if flask.request.method == "POST": envelope = common.get_return_envelope(data=response) status_code = status.HTTP_201_CREATED return flask.jsonify(**envelope), status_code # elif silent_print: # return True, {} else: if page_count or silent_print: envelope = common.get_return_envelope(False, "") else: # elif flask.request.method == "POST": envelope = common.get_return_envelope( False, "json_file is missing from your request") return flask.jsonify(**envelope), status.HTTP_400_BAD_REQUEST except Exception as e: return error("Error generating print preview, error message: " + str(e))
def print_f99_pdftk_html(stamp_print): # check if json_file is in the request # HTML("templates/forms/test.html").write_pdf("output/pdf/test/test.pdf") # HTML(string='''<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><div><b>This is bold text</b></div><div><u>This is underline text</u></div><div><i>This is italics text</i><u><br></u></div><div align='center'><u>Title</u></div><div align='left'><u><br></u></div><ol><li>one</li><li>two</li><li>three</li></ol>''').write_pdf("output/pdf/test/test.pdf") # pdfkit.from_file("templates/forms/test.html", "output/pdf/test/test.pdf") # pypdftk.stamp(current_app.config['FORM_TEMPLATES_LOCATION'].format('F99'), "output/pdf/test/test.pdf", "output/pdf/test/output.pdf") if 'json_file' in request.files: total_no_of_pages = 1 page_no = 1 json_file = request.files.get('json_file') # generate md5 for json file json_file_md5 = utils.md5_for_file(json_file) json_file.stream.seek(0) md5_directory = current_app.config['OUTPUT_DIR_LOCATION'].format( json_file_md5) os.makedirs(md5_directory, exist_ok=True) # os.makedirs(md5_directory + "images", exist_ok=True) if not os.path.exists(md5_directory + "images"): shutil.copytree("templates/forms/F99/images", md5_directory + "images") shutil.copyfile("templates/forms/F99/form-text.css", md5_directory + "form-text.css") infile = current_app.config['HTML_FORM_TEMPLATES_LOCATION'].format( 'template') json_file.save( current_app.config['REQUEST_FILE_LOCATION'].format(json_file_md5)) outfile = md5_directory + json_file_md5 + '.html' json_data = json.load( open(current_app.config['REQUEST_FILE_LOCATION'].format( json_file_md5))) form99_json_data = json_data['data'] # load the file with open(infile) as inf: txt = inf.read() soup = bs4.BeautifulSoup(txt) soup.find('label', attrs={ 'id': 'committeeName' }).string = form99_json_data['committeeName'] soup.find('label', attrs={ 'id': 'street1' }).string = form99_json_data['street1'] soup.find('label', attrs={ 'id': 'street2' }).string = form99_json_data['street2'] soup.find('label', attrs={ 'id': 'city' }).string = form99_json_data['city'] soup.find('label', attrs={ 'id': 'state' }).string = form99_json_data['state'] soup.find('label', attrs={ 'id': 'zipCode' }).string = form99_json_data['zipCode'] soup.find('span', attrs={ 'id': 'committeeId' }).string = form99_json_data['committeeId'] soup.find('label', attrs={'id': 'treasurerFullName'}).string = form99_json_data['treasurerLastName'] + \ ', ' + form99_json_data['treasurerFirstName'] \ + ', ' + form99_json_data['treasurerMiddleName'] \ + ', ' + form99_json_data['treasurerPrefix'] \ + ', ' + form99_json_data['treasurerSuffix'] soup.find('label', attrs={'id': 'treasurerName'}).string = form99_json_data['treasurerLastName'] + \ ', ' + form99_json_data['treasurerFirstName'] f99_html_data = form99_json_data['text'] soup.find('label', attrs={'id': 'text'}).string = f99_html_data soup.find('label', attrs={ 'id': form99_json_data['reason'] }).string = 'X' date_array = form99_json_data['dateSigned'].split("/") soup.find('span', attrs={ 'id': 'dateSignedMonth' }).string = str(date_array[0]) soup.find('span', attrs={ 'id': 'dateSignedDate' }).string = str(date_array[1]) soup.find('span', attrs={ 'id': 'dateSignedYear' }).string = str(date_array[2]) with open(outfile, "w") as output_file: output_file.write( str(soup).replace("<", "<").replace(">", ">")) # F99 PDF page padding options options = { 'margin-top': '0.36in', 'margin-right': '0.25in', 'margin-bottom': '0.39in', 'margin-left': '0.25in' } # HTML(outfile).write_pdf(md5_directory + json_file_md5 + '.pdf', stylesheets=[CSS(current_app.config['FORMS_LOCATION'].format('F99.css'))]) pdfkit.from_file(outfile, md5_directory + json_file_md5 + '.pdf', options=options) total_no_of_pages = pypdftk.get_num_pages(md5_directory + json_file_md5 + '.pdf') page_number_file = current_app.config[ 'FORM_TEMPLATES_LOCATION'].format('Page_Number') # checking if attachment_file exist if 'attachment_file' in request.files: # reading Attachment title file attachment_title_file = current_app.config[ 'FORM_TEMPLATES_LOCATION'].format('Attachment_Title') attachment_file = request.files.get('attachment_file') attachment_file.save( os.path.join(md5_directory + 'attachment_temp.pdf')) os.makedirs(md5_directory + 'attachment', exist_ok=True) os.makedirs(md5_directory + 'final_attachment', exist_ok=True) pypdftk.split(md5_directory + 'attachment_temp.pdf', md5_directory + 'attachment') os.remove(md5_directory + 'attachment/doc_data.txt') attachment_no_of_pages = pypdftk.get_num_pages( os.path.join(md5_directory + 'attachment_temp.pdf')) attachment_page_no = total_no_of_pages total_no_of_pages += attachment_no_of_pages # we are doing this to assign page numbers to attachment file for filename in os.listdir(md5_directory + 'attachment'): attachment_page_no += 1 pypdftk.fill_form( attachment_title_file, { "PAGESTR": "PAGE " + str(attachment_page_no) + " / " + str(total_no_of_pages) }, md5_directory + 'attachment/attachment_page_' + str(attachment_page_no) + '.pdf') pypdftk.stamp( md5_directory + 'attachment/' + filename, md5_directory + 'attachment/attachment_page_' + str(attachment_page_no) + '.pdf', md5_directory + 'final_attachment/attachment_' + str(attachment_page_no) + '.pdf') pypdftk.concat( directory_files(md5_directory + 'final_attachment/'), md5_directory + 'attachment.pdf') os.remove(md5_directory + 'attachment_temp.pdf') # shutil.rmtree(md5_directory + 'attachment') # shutil.rmtree(md5_directory + 'final_attachment') # pypdftk.concat([md5_directory + json_file_md5 + '.pdf', md5_directory + 'attachment.pdf'], md5_directory + 'all_pages_temp.pdf') # else: # shutil.move(md5_directory + json_file_md5 + '.pdf', md5_directory + 'all_pages_temp.pdf') os.makedirs(md5_directory + 'pages', exist_ok=True) os.makedirs(md5_directory + 'final_pages', exist_ok=True) pypdftk.split(md5_directory + json_file_md5 + '.pdf', md5_directory + 'pages') os.remove(md5_directory + 'pages/doc_data.txt') f99_page_no = 1 for filename in os.listdir(md5_directory + 'pages'): pypdftk.fill_form( page_number_file, { "PAGESTR": "PAGE " + str(f99_page_no) + " / " + str(total_no_of_pages) }, md5_directory + 'pages/page_number_' + str(f99_page_no) + '.pdf') pypdftk.stamp( md5_directory + 'pages/page_number_' + str(f99_page_no) + '.pdf', md5_directory + 'pages/' + filename, md5_directory + 'final_pages/page_' + str(f99_page_no) + '.pdf') f99_page_no += 1 pypdftk.concat(directory_files(md5_directory + 'final_pages/'), json_file_md5 + '_temp.pdf') if 'attachment_file' in request.files: pypdftk.concat([ json_file_md5 + '_temp.pdf', md5_directory + 'attachment.pdf' ], md5_directory + 'all_pages.pdf') shutil.rmtree(md5_directory + 'attachment') shutil.rmtree(md5_directory + 'final_attachment') os.remove(md5_directory + 'attachment.pdf') else: shutil.move(json_file_md5 + '_temp.pdf', md5_directory + 'all_pages.pdf') # clean up task shutil.rmtree(md5_directory + 'pages') shutil.rmtree(md5_directory + 'final_pages') # os.remove(md5_directory + json_file_md5 + '.html') # shutil.rmtree(md5_directory + 'images') # os.remove(md5_directory + 'form-text.css') os.remove(md5_directory + json_file_md5 + '.pdf') # for f99_page_no in range(f99_no_of_pages): # pypdftk.fill_form(page_number_file, # {"PAGESTR": "PAGE " + str(f99_page_no+1) + " / " + str(total_no_of_pages)}, # md5_directory + 'pages/page_' + str(f99_page_no+1) + '.pdf') # pypdftk.stamp(md5_directory + json_file_md5 + '.pdf', md5_directory + # 'pages/page_' + str(f99_page_no+1) + '.pdf', md5_directory + json_file_md5 + '_temp.pdf') # json_data['PAGESTR'] = "PAGE " + str(page_no) + " / " + str(total_no_of_pages) # json_data['MISCELLANEOUS_TEXT'] = '' # xfdf_path = pypdftk.gen_xfdf(json_data) # pypdftk.fill_form(infile, json_data, outfile) # HTML(string='''<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><font face='Helvetica' size=10 ''' + f99_full_text).\ # write_pdf("output/pdf/test/test.pdf") # pypdftk.stamp(outfile, "output/pdf/test/test.pdf", "output/pdf/test/output.pdf") # additional_page_counter = 0 # if len(f99_pages_text_json['additional_pages']) > 0: # continuation_file = current_app.config['FORM_TEMPLATES_LOCATION'].format('F99_CONT') # os.makedirs(md5_directory + 'merge', exist_ok=True) # for additional_page in f99_pages_text_json['additional_pages']: # page_no += 1 # continuation_outfile = md5_directory + 'merge/' + str(additional_page_counter)+'.pdf' # pypdftk.fill_form(continuation_file, {"PAGESTR": "PAGE "+str(page_no)+" / " + str(total_no_of_pages), # "CONTINOUS_TEXT": additional_page[str(additional_page_counter)]}, continuation_outfile) # pypdftk.concat([outfile, continuation_outfile], md5_directory + json_file_md5 + '_all_pages_temp.pdf') # shutil.copy(md5_directory + json_file_md5 + '_all_pages_temp.pdf', outfile) # additional_page_counter += 1 # os.remove(md5_directory + json_file_md5 + '_all_pages_temp.pdf') # # # Add the F99 attachment # if 'attachment_file' in request.files: # pypdftk.concat([outfile, md5_directory + 'attachment.pdf'], md5_directory + 'all_pages.pdf') # os.remove(md5_directory + 'attachment.pdf') # else: # shutil.copy(outfile, md5_directory + 'all_pages.pdf') # os.remove(md5_directory + json_file_md5 +'_temp.pdf') # push output file to AWS s3 = boto3.client('s3') s3.upload_file( md5_directory + 'all_pages.pdf', current_app.config['AWS_FECFILE_COMPONENTS_BUCKET_NAME'], md5_directory + 'all_pages.pdf', ExtraArgs={ 'ContentType': "application/pdf", 'ACL': "public-read" }) response = { # 'file_name': '{}.pdf'.format(json_file_md5), 'pdf_url': current_app.config['PRINT_OUTPUT_FILE_URL'].format(json_file_md5) + 'all_pages.pdf' } if flask.request.method == "POST": envelope = common.get_return_envelope(data=response) status_code = status.HTTP_201_CREATED return flask.jsonify(**envelope), status_code else: if flask.request.method == "POST": envelope = common.get_return_envelope( 'false', 'JSON file is missing from your request') status_code = status.HTTP_400_BAD_REQUEST return flask.jsonify(**envelope), status_code
def print_f99_pdftk(stamp_print): # check if json_file is in the request if 'json_file' in request.files: total_no_of_pages = 1 page_no = 1 json_file = request.files.get('json_file') # generate md5 for json file json_file_md5 = utils.md5_for_file(json_file) json_file.stream.seek(0) md5_directory = current_app.config['OUTPUT_DIR_LOCATION'].format( json_file_md5) os.makedirs(md5_directory, exist_ok=True) infile = current_app.config['FORM_TEMPLATES_LOCATION'].format('F99') # save json file as md5 file name json_file.save( current_app.config['REQUEST_FILE_LOCATION'].format(json_file_md5)) outfile = md5_directory + json_file_md5 + '_temp.pdf' json_data = json.load( open(current_app.config['REQUEST_FILE_LOCATION'].format( json_file_md5))) # setting timestamp and imgno to empty as these needs to show up after submission if stamp_print != 'stamp': json_data['FILING_TIMESTAMP'] = '' json_data['IMGNO'] = '' f99_pages_text_json = json.loads(split_f99_text_pages(json_data)) json_data['MISCELLANEOUS_TEXT'] = f99_pages_text_json['main_page'] total_no_of_pages += len(f99_pages_text_json['additional_pages']) # checking if attachment_file exist if 'attachment_file' in request.files: # reading Attachment title file attachment_title_file = current_app.config[ 'FORM_TEMPLATES_LOCATION'].format('Attachment_Title') attachment_file = request.files.get('attachment_file') attachment_file.save( os.path.join(md5_directory + 'attachment_temp.pdf')) os.makedirs(md5_directory + 'attachment', exist_ok=True) os.makedirs(md5_directory + 'final_attachment', exist_ok=True) pypdftk.split(md5_directory + 'attachment_temp.pdf', md5_directory + 'attachment') os.remove(md5_directory + 'attachment/doc_data.txt') attachment_no_of_pages = pypdftk.get_num_pages( os.path.join(md5_directory + 'attachment_temp.pdf')) attachment_page_no = total_no_of_pages total_no_of_pages += attachment_no_of_pages # we are doing this to assign page numbers to attachment file for filename in os.listdir(md5_directory + 'attachment'): attachment_page_no += 1 pypdftk.fill_form( attachment_title_file, { "PAGESTR": "PAGE " + str(attachment_page_no) + " / " + str(total_no_of_pages) }, md5_directory + 'attachment/attachment_page_' + str(attachment_page_no) + '.pdf') pypdftk.stamp( md5_directory + 'attachment/' + filename, md5_directory + 'attachment/attachment_page_' + str(attachment_page_no) + '.pdf', md5_directory + 'final_attachment/attachment_' + str(attachment_page_no) + '.pdf') pypdftk.concat( directory_files(md5_directory + 'final_attachment/'), md5_directory + 'attachment.pdf') os.remove(md5_directory + 'attachment_temp.pdf') shutil.rmtree(md5_directory + 'attachment') shutil.rmtree(md5_directory + 'final_attachment') json_data['PAGESTR'] = "PAGE " + str(page_no) + " / " + str( total_no_of_pages) pypdftk.fill_form(infile, json_data, outfile, flatten=False) additional_page_counter = 0 if len(f99_pages_text_json['additional_pages']) > 0: continuation_file = current_app.config[ 'FORM_TEMPLATES_LOCATION'].format('F99_CONT') os.makedirs(md5_directory + 'merge', exist_ok=True) for additional_page in f99_pages_text_json['additional_pages']: page_no += 1 continuation_outfile = md5_directory + 'merge/' + str( additional_page_counter) + '.pdf' pypdftk.fill_form( continuation_file, { "PAGESTR": "PAGE " + str(page_no) + " / " + str(total_no_of_pages), "CONTINOUS_TEXT": additional_page[str(additional_page_counter)] }, continuation_outfile) pypdftk.concat([outfile, continuation_outfile], md5_directory + json_file_md5 + '_all_pages_temp.pdf') shutil.copy( md5_directory + json_file_md5 + '_all_pages_temp.pdf', outfile) additional_page_counter += 1 os.remove(md5_directory + json_file_md5 + '_all_pages_temp.pdf') # Add the F99 attachment if 'attachment_file' in request.files: pypdftk.concat([outfile, md5_directory + 'attachment.pdf'], md5_directory + 'all_pages.pdf') os.remove(md5_directory + 'attachment.pdf') else: shutil.copy(outfile, md5_directory + 'all_pages.pdf') os.remove(md5_directory + json_file_md5 + '_temp.pdf') # push output file to AWS s3 = boto3.client('s3') s3.upload_file( md5_directory + 'all_pages.pdf', current_app.config['AWS_FECFILE_COMPONENTS_BUCKET_NAME'], md5_directory + 'all_pages.pdf', ExtraArgs={ 'ContentType': "application/pdf", 'ACL': "public-read" }) response = { # 'file_name': '{}.pdf'.format(json_file_md5), 'pdf_url': current_app.config['PRINT_OUTPUT_FILE_URL'].format(json_file_md5) + 'all_pages.pdf' } if flask.request.method == "POST": envelope = common.get_return_envelope(data=response) status_code = status.HTTP_201_CREATED return flask.jsonify(**envelope), status_code else: if flask.request.method == "POST": envelope = common.get_return_envelope( 'false', 'JSON file is missing from your request') status_code = status.HTTP_400_BAD_REQUEST return flask.jsonify(**envelope), status_code
def print_f99(): """ This function is being invoked internally from controllers HTTP request needs to have form_type, file, and attachment_file form_type : F99 json_file: please refer to below sample JSON attachment_file: It is a PDF file that will be merged to the generated PDF file. sample: { "REASON_TYPE":"MST", "COMMITTEE_NAME":"DONALD J. TRUMP FOR PRESIDENT, INC.", "FILER_FEC_ID_NUMBER":"C00580100", "IMGNO":"201812179143565008", "FILING_TIMESTAMP":"12/17/2018 17 : 09", "STREET_1":"725 FIFTH AVENUE", "STREET_2":"", "CITY":"NEW YORK", "STATE":"NY", "ZIP":"10022", "TREASURER_FULL_NAME":"CRATE, BRADLEY, , ,", "TREASURER_NAME":"CRATE, BRADLEY, , ,", "EF_STAMP":"[Electronically Filed]", "DATE_SIGNED_MM":"01", "DATE_SIGNED_DD":"28", "DATE_SIGNED_YY":"2019", "MISCELLANEOUS_TEXT":"This statement is in response to the Commission's letter to the Committee dated November 12, 2018, regarding two items related to the above-referenced report ('the Original Report')." } :return: return JSON response sample: { "message": "", "results": { "file_name": "bd78435a70a70d656145dae89e0e22bb.pdf", "file_url": "https://fecfile-dev-components.s3.amazonaws.com/output/bd78435a70a70d656145dae89e0e22bb.pdf" }, "success": "true" } """ if 'json_file' in request.files: json_file = request.files.get('json_file') json_file_md5 = utils.md5_for_file(json_file) json_file.stream.seek(0) infile = current_app.config['FORM_TEMPLATES_LOCATION'].format('F99') json_file.save( current_app.config['REQUEST_FILE_LOCATION'].format(json_file_md5)) outfile = current_app.config['OUTPUT_FILE_LOCATION'].format( json_file_md5) json_data = json.load( open(current_app.config['REQUEST_FILE_LOCATION'].format( json_file_md5))) # json_data['FILER_FEC_ID_NUMBER'] = json_data['FILER_FEC_ID_NUMBER'][1:] if json_data['REASON_TYPE'] == 'MST': reason_type_data = {"REASON_TYPE_MST": "/MST"} if json_data['REASON_TYPE'] == 'MSM': reason_type_data = {"REASON_TYPE_MSM": "/MSM"} if json_data['REASON_TYPE'] == 'MSI': reason_type_data = {"REASON_TYPE_MSI": "/MSI"} if json_data['REASON_TYPE'] == 'MSW': reason_type_data = {"REASON_TYPE_MSW": "/MSW"} # open the input file input_stream = open(infile, "rb") pdf_reader = PdfFileReader(input_stream, strict=True) if "/AcroForm" in pdf_reader.trailer["/Root"]: pdf_reader.trailer["/Root"]["/AcroForm"].update( {NameObject("/NeedAppearances"): BooleanObject(True)}) pdf_writer = PdfFileWriter() form.set_need_appearances_writer(pdf_writer) if "/AcroForm" in pdf_writer._root_object: pdf_writer._root_object["/AcroForm"].update( {NameObject("/NeedAppearances"): BooleanObject(True)}) for page_num in range(pdf_reader.numPages): page_obj = pdf_reader.getPage(page_num) pdf_writer.addPage(page_obj) form.update_checkbox_values(page_obj, reason_type_data) pdf_writer.updatePageFormFieldValues(page_obj, json_data) # Add the F99 attachment if 'attachment_file' in request.files: # reading Attachment title file attachment_title_file = current_app.config[ 'FORM_TEMPLATES_LOCATION'].format('Attachment_Title') attachment_title_stream = open(attachment_title_file, "rb") attachment_title_reader = PdfFileReader(attachment_title_stream, strict=True) attachment_stream = request.files.get('attachment_file') attachment_reader = PdfFileReader(attachment_stream, strict=True) for attachment_page_num in range(attachment_reader.numPages): attachment_page_obj = attachment_reader.getPage( attachment_page_num) if attachment_page_num == 0: attachment_page_obj.mergePage( attachment_title_reader.getPage(0)) pdf_writer.addPage(attachment_page_obj) output_stream = open(outfile, "wb") pdf_writer.write(output_stream) input_stream.close() output_stream.close() # push output file to AWS s3 = boto3.client('s3') s3.upload_file( outfile, current_app.config['AWS_FECFILE_COMPONENTS_BUCKET_NAME'], current_app.config['OUTPUT_FILE_LOCATION'].format(json_file_md5), ExtraArgs={ 'ContentType': "application/pdf", 'ACL': "public-read" }) response = { # 'file_name': '{}.pdf'.format(json_file_md5), 'pdf_url': current_app.config['PRINT_OUTPUT_FILE_URL'].format(json_file_md5) } if flask.request.method == "POST": envelope = common.get_return_envelope(data=response) status_code = status.HTTP_201_CREATED return flask.jsonify(**envelope), status_code else: if flask.request.method == "POST": envelope = common.get_return_envelope( 'false', 'JSON file is missing from your request') status_code = status.HTTP_400_BAD_REQUEST return flask.jsonify(**envelope), status_code
def parse_next_in_image_generator_queue(): """***************************************************************************************************************** Function will get the next report to stamp image on PDF ******************************************************************************************************************** 1. Image Generator worker will call this function on a set interval, say every 15 seconds 2. This Function will read the message from the SQS one at a time and sets the visibility to N number of minutes so that other workers will not see it. Sample Message message_attributes = {'submissionId': {'StringValue': submission_id, 'DataType': 'String'}, 'committeeId': {'StringValue': committee_id, 'DataType': 'String'}, 'fileName': {'StringValue': file_name, 'DataType': 'String'}, 'receivedTime': {'StringValue': str(upload_time), 'DataType': 'String'}, 'beginImageNumber': {'StringValue': begin_image_number, 'DataType': 'String'} } *****************************************************************************************************************""" sqs = boto3.resource("sqs") queue = sqs.get_queue_by_name(QueueName=cfg.IMAGE_GENERATOR_SQS_QUEUE) try: # Getting One message at a time messages = queue.receive_messages( MaxNumberOfMessages=1, MessageAttributeNames=["All"], VisibilityTimeout=cfg.MESSAGE_VISIBILITY_TIMEOUT, ) except Exception as e: envelope = common.get_return_envelope( "false", "unable to read message from image generator queue") return flask.jsonify(**envelope), status.HTTP_400_BAD_REQUEST # next_image_generator = [] # next_image_generator.append("one") # res = image_generator_data(next_image_generator) # print(res) # return res if len(messages) > 0: # Getting the first message for message in messages: receipt_handle = message.receipt_handle # process the messages msg_body = message.body next_image_generator = [] print( "***************************************************************************************************" ) print("Getting Message from the SQS: " + str(msg_body)) print(message.message_attributes) print( "***************************************************************************************************" ) if message.message_attributes is not None: next_image_generator.append({ "submissionId": message.message_attributes.get("submissionId").get( "StringValue"), "committeeId": message.message_attributes.get("committeeId").get( "StringValue"), "fileName": message.message_attributes.get("fileName").get( "StringValue"), "beginImageNumber": message.message_attributes.get("beginImageNumber").get( "StringValue"), "receivedTime": message.message_attributes.get("receivedTime").get( "StringValue"), }) # Parsing the data res = image_generator_data(next_image_generator[0]) message.delete() print(res) return res else: print("Nothing to process - Message Queue is empty") envelope = common.get_return_envelope( "true", "Nothing to process - Message Queue is empty") return flask.jsonify(**envelope), status.HTTP_200_OK
def parse_next_filing_from_image_number_queue(): """***************************************************************************************************************** Function will get the next report to image from imaging queue and processes it ******************************************************************************************************************** 1. Imaging worker will call this function on a set interval, say every 15 seconds 2. This Function will read the message from the SQS one at a time and sets the visibility to N number of minutes so that other workers will not see it. Sample Message message_attributes = {'submissionId': {'StringValue': submission_id, 'DataType': 'String'}, 'committeeId': {'StringValue': committee_id, 'DataType': 'String'}, 'fileName': {'StringValue': file_name, 'DataType': 'String'}, 'receivedTime': {'StringValue': str(upload_time), 'DataType': 'String'}, 'beginImageNumber': {'StringValue': begin_image_number, 'DataType': 'String'} } *****************************************************************************************************************""" sqs = boto3.resource("sqs") queue = sqs.get_queue_by_name(QueueName=cfg.IMAGE_NUMBER_SQS_QUEUE) try: # Getting One message at a time messages = queue.receive_messages( MaxNumberOfMessages=1, MessageAttributeNames=["All"], VisibilityTimeout=cfg.MESSAGE_VISIBILITY_TIMEOUT, ) except Exception as e: envelope = common.get_return_envelope( "false", "unable to read message from image number queue") return flask.jsonify(**envelope), status.HTTP_400_BAD_REQUEST # messages = {} # next_imaging = [] # next_imaging.append({"submissionId":"fab1d0fc-0089-4b47-8b80-0a1f3f970066","committeeId":"C00337733", # "fileName": "C00337733_fab1d0fc00894b478b800a1f3f970066.json","beginImageNumber":""}) # image_number = image_number_data(next_imaging[0]) # message.delete() # print(image_number) if len(messages) > 0: # Getting the first message for message in messages: receipt_handle = message.receipt_handle # process the messages msg_body = message.body next_imaging = [] print( "***************************************************************************************************" ) print("Getting Message from the SQS: " + str(msg_body)) print(message.message_attributes) print( "***************************************************************************************************" ) if message.message_attributes is not None: next_imaging.append({ "submissionId": message.message_attributes.get("submissionId").get( "StringValue"), "committeeId": message.message_attributes.get("committeeId").get( "StringValue"), "fileName": message.message_attributes.get("fileName").get( "StringValue"), "beginImageNumber": message.message_attributes.get("beginImageNumber").get( "StringValue"), }) # Parsing the data image_number = image_number_data(next_imaging[0]) message.delete() print(image_number) # return res res = flask.jsonify( {"result": [{ "beginImageNum": str(image_number) }]}) return res else: print("Nothing to process - Message Queue is empty") envelope = common.get_return_envelope( "true", "Nothing to process - Message Queue is empty") # next_imaging = [] # next_imaging.append( # { # "submissionId": "374d7ea3-1718-4763-b9f3-fee88acecc3c", # "committeeId": "C00024679", # "fileName": "C00024679_374d7ea317184763b9f3fee88acecc3c.json", # "beginImageNumber": "20201109000000" # } # ) # image_number_data(next_imaging) return flask.jsonify(**envelope), status.HTTP_200_OK
def paginate(file_content=None, begin_image_num=None): if file_content and begin_image_num: txn_img_num = begin_image_num # if "json_file_name" in request.json: # json_file_name = request.json.get("json_file_name") # txn_img_num = request.json.get("begin_image_num") # if not txn_img_num: # if flask.request.method == "POST": # envelope = common.get_return_envelope( # "false", "begin_image_num is missing from your request" # ) # status_code = status.HTTP_400_BAD_REQUEST # return flask.jsonify(**envelope), status_code # # file_url = current_app.config["AWS_S3_FECFILE_COMPONENTS_DOMAIN"] + "/" + json_file_name + ".json" # file_url = "https://dev-efile-repo.s3.amazonaws.com/" + json_file_name + ".json" # with urllib.request.urlopen(file_url) as url: # file_content = url.read().decode() f24_json = json.loads(file_content) data = f24_json["data"] txn_img_json = {} total_no_of_pages = 0 if not data.get("memoText") or not data.get("reportPrint"): txn_img_num -= 1 if data.get("schedules") and data["schedules"].get("SE"): map_txn_img_num( schedules=data["schedules"]["SE"], num=2, txn_img_json=txn_img_json, image_num=txn_img_num, ) total_no_of_pages = get_total_pages(data) summary = {} if txn_img_json: summary["begin_image_num"] = min(txn_img_json.values()) summary["end_image_num"] = max(txn_img_json.values()) else: summary["begin_image_num"] = begin_image_num summary["end_image_num"] = txn_img_num summary["committeeId"] = data.get("committeeId", None) txn_img_json["summary"] = summary # return True, {"total_pages": total_no_of_pages, "txn_img_json": txn_img_json} response = { "total_pages": total_no_of_pages, "txn_img_json": txn_img_json } # if flask.request.method == "POST": envelope = common.get_return_envelope(data=response) return flask.jsonify(**envelope), status.HTTP_200_OK else: # return False, None # if flask.request.method == "POST": envelope = common.get_return_envelope( False, "json_file_name is missing from your request") return flask.jsonify(**envelope), status.HTTP_400_BAD_REQUEST
def print_pdftk( stamp_print="", page_count=False, file_content=None, begin_image_num=None, silent_print=False, filing_timestamp=None, rep_id=None, ): # try: silent_print = silent_print txn_img_num = begin_image_num filing_timestamp = filing_timestamp if "json_file" in request.files or (page_count and file_content): # check if json_file_name is in the request if "json_file" in request.files: json_file = request.files.get("json_file") page_count = page_count silent_print = (True if request.form.get("silent_print") and request.form.get("silent_print").lower() in ["true", "1"] else False) page_count = (True if request.form.get("page_count") and request.form.get("page_count").lower() in ["true", "1"] else False) if silent_print: txn_img_num = request.form.get("begin_image_num", None) if not txn_img_num: if flask.request.method == "POST": envelope = common.get_return_envelope( "false", "begin_image_num is missing from your request") status_code = status.HTTP_400_BAD_REQUEST return flask.jsonify(**envelope), status_code txn_img_num = int(txn_img_num) filing_timestamp = request.form.get("filing_timestamp", None) json_file_md5 = md5_for_file(json_file) json_file.stream.seek(0) # save json file as md5 file name json_file.save(current_app.config["REQUEST_FILE_LOCATION"].format( json_file_md5)) # load json file f24_json = json.load( open(current_app.config["REQUEST_FILE_LOCATION"].format( json_file_md5))) # if page_count is True then return from here elif page_count and file_content: f24_json = json.loads(file_content) # return {"total_pages": get_total_pages(f24_json.get("data"))} response = {"total_pages": get_total_pages(f24_json.get("data"))} # if flask.request.method == "POST": envelope = common.get_return_envelope(data=response) return flask.jsonify(**envelope), status.HTTP_200_OK elif silent_print and begin_image_num and file_content: json_file_md5 = md5_for_text(file_content) f24_json = json.loads(file_content) md5_directory = current_app.config["OUTPUT_DIR_LOCATION"].format( json_file_md5) # deleting directory if it exists and has any content delete_directory(md5_directory) os.makedirs(md5_directory, exist_ok=True) # setting timestamp and imgno to empty as these needs to show up after submission output = {} if stamp_print != "stamp": output["FILING_TIMESTAMP"] = "" output["IMGNO"] = "" # read data from json file f24_data = f24_json["data"] reportId = str(f24_data["reportId"]) os.makedirs(md5_directory + reportId + "/", exist_ok=True) output["committeeId"] = f24_data["committeeId"] output["committeeName"] = f24_data["committeeName"] output["reportType"] = f24_data["reportType"] output["amendIndicator"] = f24_data["amendIndicator"] output["efStamp"] = "[Electronically Filed]" if output["amendIndicator"] == "A": if f24_data["amendDate"]: amend_date_array = f24_data["amendDate"].split("/") output["amendDate_MM"] = amend_date_array[0] output["amendDate_DD"] = amend_date_array[1] output["amendDate_YY"] = amend_date_array[2] se_count = 0 # Calculating total number of pages if not f24_data["schedules"].get("SE"): output["PAGENO"] = 1 output["TOTALPAGES"] = 1 else: se_count = len(f24_data["schedules"]["SE"]) output["TOTALPAGES"] = get_total_pages(f24_data) # make it true when filing_timestamp has been passed for the first time is_file_timestamp = False # Printing report memo text page if f24_data.get("memoText") and f24_data.get("reportPrint"): memo_dict = { "scheduleName_1": "F3X" + f24_data["amendIndicator"], "memoDescription_1": f24_data["memoText"], "PAGESTR": "PAGE " + str(1) + " / " + str(output["TOTALPAGES"]), } if silent_print: memo_dict["IMGNO"] = txn_img_num txn_img_num += 1 if filing_timestamp: memo_dict["FILING_TIMESTAMP"] = filing_timestamp is_file_timestamp = True print_summary(memo_dict, 1, reportId, json_file_md5) if f24_data.get("filedDate"): filed_date_array = f24_data["filedDate"].split("/") output["filedDate_MM"] = filed_date_array[0] output["filedDate_DD"] = filed_date_array[1] output["filedDate_YY"] = filed_date_array[2] # build treasurer name to map it to PDF template treasurerFullName = "" for item in name_list: item = "treasurer" + item if f24_data.get(item): treasurerFullName += f24_data.get(item) + ", " output["treasurerFullName"] = treasurerFullName[:-2] output["treasurerName"] = (f24_data.get("treasurerLastName", "") + ", " + f24_data.get("treasurerFirstName", "")) output["treasurerName"] = ( output["treasurerName"].strip().rstrip(",").strip()) if f24_data["schedules"].get("SE"): page_index = (2 if f24_data.get("memoText") and f24_data.get("reportPrint") else 1) page_dict = {} sub_total = 0 total = 0 for i, se in enumerate(f24_data["schedules"]["SE"]): index = (i % 2) + 1 if se.get("payeeLastName"): payeeName = "" for item in name_list: item = "payee" + item if se.get(item): payeeName += se.get(item) + ", " page_dict["payeeName_" + str(index)] = payeeName[:-2] elif se.get("payeeOrganizationName"): page_dict["payeeName_" + str(index)] = se["payeeOrganizationName"] page_dict["memoCode_" + str(index)] = se["memoCode"] page_dict["memoDescription_" + str(index)] = se["memoDescription"] page_dict["payeeStreet1_" + str(index)] = se["payeeStreet1"] page_dict["payeeStreet2_" + str(index)] = se["payeeStreet2"] page_dict["payeeCity_" + str(index)] = se["payeeCity"] page_dict["payeeState_" + str(index)] = se["payeeState"] page_dict["payeeZipCode_" + str(index)] = se["payeeZipCode"] page_dict["expenditureAmount_" + str(index)] = "{:.2f}".format( se["expenditureAmount"]) page_dict["transactionId_" + str(index)] = se["transactionId"] page_dict["expenditurePurpose_" + str(index)] = se["expenditurePurposeDescription"] page_dict["supportOppose_" + str(index)] = se["support/opposeCode"] page_dict["candidateOffice_" + str(index)] = se["candidateOffice"] page_dict["candidateState_" + str(index)] = se["candidateState"] page_dict["candidateDistrict_" + str(index)] = se["candidateDistrict"] page_dict["electionType_" + str(index)] = se["electionCode"][:1] page_dict["electionYear_" + str(index)] = se["electionCode"][1:] page_dict["electionOtherDescription_" + str(index)] = se["electionOtherDescription"] page_dict["calendarYTD_" + str(index)] = "{:.2f}".format( se["calendarYTDPerElectionForOffice"]) if se.get("disseminationDate"): dissem_date_array = se["disseminationDate"].split("/") page_dict["disseminationDate_MM_" + str(index)] = dissem_date_array[0] page_dict["disseminationDate_DD_" + str(index)] = dissem_date_array[1] page_dict["disseminationDate_YY_" + str(index)] = dissem_date_array[2] if se.get("disbursementDate"): disburse_date_array = se["disbursementDate"].split("/") page_dict["disbursementDate_MM_" + str(index)] = disburse_date_array[0] page_dict["disbursementDate_DD_" + str(index)] = disburse_date_array[1] page_dict["disbursementDate_YY_" + str(index)] = disburse_date_array[2] candidateName = "" for item in name_list: item = "candidate" + item if se.get(item): candidateName += se.get(item) + ", " if candidateName: page_dict["candidateName_" + str(index)] = candidateName[:-2] else: page_dict["candidateName_" + str(index)] = "" # if se[item]: candidate_name_list.append(se[item]) # page_dict["candidateName_" + str(index)] = " ".join(candidate_name_list) if se.get("memoCode") != "X": sub_total += se["expenditureAmount"] total += se["expenditureAmount"] # print and reset if index % 2 == 0 or i == se_count - 1: page_dict["PAGENO"] = page_index page_dict["subTotal"] = "{:.2f}".format(sub_total) if silent_print: page_dict["IMGNO"] = txn_img_num txn_img_num += 1 if filing_timestamp and not is_file_timestamp: page_dict["FILING_TIMESTAMP"] = filing_timestamp is_file_timestamp = True if i == se_count - 1: page_dict["total"] = "{:.2f}".format(total) print_dict = {**output, **page_dict} print_f24(print_dict, page_index, reportId, json_file_md5) page_index += 1 memo_dict = {} for xir in range(1, 3): if page_dict.get("memoDescription_{}".format(xir)): memo_dict["scheduleName_{}".format(xir)] = "SE" memo_dict["memoDescription_{}".format( xir)] = page_dict["memoDescription_{}".format( xir)] memo_dict["transactionId_{}".format( xir)] = page_dict["transactionId_{}".format( xir)] memo_dict["PAGESTR"] = ("PAGE " + str(page_index) + " / " + str(output["TOTALPAGES"])) if silent_print: memo_dict["IMGNO"] = txn_img_num if silent_print: txn_img_num += 1 if memo_dict: print_summary(memo_dict, page_index, reportId, json_file_md5) page_index += 1 page_dict = {} sub_total = 0 else: output["subTotal"] = "0.00" output["total"] = "0.00" print_f24(output, 1, reportId, json_file_md5) # Concatenating all generated pages for i in range(1, output["TOTALPAGES"] + 1, 1): if path.isfile(md5_directory + reportId + "/F24_temp.pdf"): pypdftk.concat( [ md5_directory + reportId + "/F24_temp.pdf", md5_directory + reportId + "/F24_{}.pdf".format(i), ], md5_directory + reportId + "/concat_F24.pdf", ) os.rename( md5_directory + reportId + "/concat_F24.pdf", md5_directory + reportId + "/F24_temp.pdf", ) os.remove(md5_directory + reportId + "/F24_{}.pdf".format(i)) else: os.rename( md5_directory + reportId + "/F24_{}.pdf".format(i), md5_directory + reportId + "/F24_temp.pdf", ) os.rename( md5_directory + reportId + "/F24_temp.pdf", md5_directory + reportId + "/F24.pdf", ) response = { "total_pages": output["TOTALPAGES"], } if not page_count: s3 = boto3.client("s3") extraArgs = { "ContentType": "application/pdf", "ACL": "public-read" } if silent_print: response["pdf_url"] = ( current_app.config["AWS_FECFILE_COMPONENTS_BUCKET_NAME"], rep_id + ".pdf", ) s3.upload_file( md5_directory + reportId + "/F24.pdf", current_app.config["AWS_FECFILE_COMPONENTS_BUCKET_NAME"], rep_id + ".pdf", ExtraArgs=extraArgs, ) else: response["pdf_url"] = ( current_app.config["PRINT_OUTPUT_FILE_URL"].format( json_file_md5) + "F24.pdf", ) s3.upload_file( md5_directory + reportId + "/F24.pdf", current_app.config["AWS_FECFILE_COMPONENTS_BUCKET_NAME"], md5_directory + "F24.pdf", ExtraArgs=extraArgs, ) # if flask.request.method == "POST": envelope = common.get_return_envelope(data=response) return flask.jsonify(**envelope), status.HTTP_201_CREATED else: if page_count or silent_print: envelope = common.get_return_envelope(False, "") elif flask.request.method == "POST": envelope = common.get_return_envelope( False, "json_file is missing from your request") return flask.jsonify(**envelope), status.HTTP_400_BAD_REQUEST
def print_pdftk(stamp_print): # check if json_file is in the request # try: if 'json_file' in request.files: total_no_of_pages = 0 page_no = 1 has_sa_schedules = has_sb_schedules = False json_file = request.files.get('json_file') # generate md5 for json file # FIXME: check if PDF already exist with md5, if exist return pdf instead of re-generating PDF file. json_file_md5 = utils.md5_for_file(json_file) json_file.stream.seek(0) md5_directory = current_app.config['OUTPUT_DIR_LOCATION'].format(json_file_md5) os.makedirs(md5_directory, exist_ok=True) infile = current_app.config['FORM_TEMPLATES_LOCATION'].format('F3L') # save json file as md5 file name json_file.save(current_app.config['REQUEST_FILE_LOCATION'].format(json_file_md5)) outfile = md5_directory + json_file_md5 + '_temp.pdf' # load json file f3l_json = json.load(open(current_app.config['REQUEST_FILE_LOCATION'].format(json_file_md5))) # setting timestamp and imgno to empty as these needs to show up after submission output = {} if stamp_print != 'stamp': output['FILING_TIMESTAMP'] = '' output['IMGNO'] = '' # read data from json file f3l_data = f3l_json['data'] # check if summary is present in fecDataFile f3l_summary = [] if 'summary' in f3l_data: f3l_summary = f3l_data['summary'] # split coverage start date and coverage end date to set month, day, and year if f3l_data['coverageStartDate'] and len(f3l_data['coverageStartDate']) > 0: coverage_start_date_array = f3l_data['coverageStartDate'].split("/") f3l_data['coverageStartDateMonth'] = coverage_start_date_array[0] f3l_data['coverageStartDateDay'] = coverage_start_date_array[1] f3l_data['coverageStartDateYear'] = coverage_start_date_array[2] if f3l_data['coverageEndDate'] and len(f3l_data['coverageEndDate']) > 0: coverage_end_date_array = f3l_data['coverageEndDate'].split("/") f3l_data['coverageEndDateMonth'] = coverage_end_date_array[0] f3l_data['coverageEndDateDay'] = coverage_end_date_array[1] f3l_data['coverageEndDateYear'] = coverage_end_date_array[2] # checking for signed date, it is only available for submitted reports if f3l_data['date_signed'] and len(f3l_data['date_signed']) > 0: date_signed_array = f3l_data['date_signed'].split("-") f3l_data['dateSignedMonth'] = date_signed_array[0] f3l_data['dateSignedDay'] = date_signed_array[1] f3l_data['dateSignedYear'] = date_signed_array[2] # build treasurer name to map it to PDF template treasurer_full_name = [] treasurer_full_name.append(f3l_data['treasurerLastName']) treasurer_full_name.append(f3l_data['treasurerFirstName']) treasurer_full_name.append(f3l_data['treasurerMiddleName']) treasurer_full_name.append(f3l_data['treasurerPrefix']) treasurer_full_name.append(f3l_data['treasurerSuffix']) f3l_data['treasurerFullName'] = ",".join(map(str, treasurer_full_name)) f3l_data['treasurerName'] = f3l_data['treasurerLastName'] + "," + f3l_data['treasurerFirstName'] f3l_data['efStamp'] = '[Electronically Filed]' # checking if json contains summary details, for individual transactions print there wouldn't be summary if len(f3l_summary) > 0: total_no_of_pages = 1 f3l_data_summary_array = [f3l_data, f3l_summary] if 'memoText' in f3l_data and f3l_data['memoText']: total_no_of_pages += 1 else: f3l_data_summary_array = [f3l_data] f3l_data_summary = {i: j for x in f3l_data_summary_array for i, j in x.items()} # process all schedules and build the PDF's process_output, total_no_of_pages = process_schedules(f3l_data, md5_directory, total_no_of_pages) has_sa_schedules = process_output.get('has_sa_schedules') has_sb_schedules = process_output.get('has_sb_schedules') if len(f3l_summary) > 0: get_summary_detail(f3l_summary, f3l_data, f3l_data_summary) f3l_data_summary['PAGESTR'] = "PAGE " + str(page_no) + " / " + str(total_no_of_pages) pypdftk.fill_form(infile, f3l_data_summary, outfile) shutil.copy(outfile, md5_directory + 'F3L_Summary.pdf') os.remove(md5_directory + json_file_md5 + '_temp.pdf') # Memo text changes if 'memoText' in f3l_data_summary and f3l_data_summary['memoText']: memo_dict = {} temp_memo_outfile = md5_directory + 'F3L_Summary_memo.pdf' memo_infile = current_app.config['FORM_TEMPLATES_LOCATION'].format('TEXT') memo_dict['scheduleName_1'] = 'F3L' + f3l_data_summary['amendmentIndicator'] memo_dict['memoDescription_1'] = f3l_data_summary['memoText'] memo_dict['PAGESTR'] = "PAGE " + str(2) + " / " + str(total_no_of_pages) pypdftk.fill_form(memo_infile, memo_dict, temp_memo_outfile) pypdftk.concat([md5_directory + 'F3L_Summary.pdf', temp_memo_outfile], md5_directory + json_file_md5 + '_temp.pdf') shutil.copy(md5_directory + json_file_md5 + '_temp.pdf', md5_directory + 'F3L_Summary.pdf') os.remove(md5_directory + json_file_md5 + '_temp.pdf') # check if all_pages already exsits if os.path.exists(md5_directory + 'all_pages.pdf'): os.remove(md5_directory + 'all_pages.pdf') # checking for sa transactions if has_sa_schedules: pypdftk.concat([md5_directory + 'F3L_Summary.pdf', md5_directory + 'SA/all_pages.pdf'], md5_directory + 'all_pages.pdf') os.remove(md5_directory + 'SA/all_pages.pdf') shutil.rmtree(md5_directory + 'SA') else: shutil.copy(md5_directory + 'F3L_Summary.pdf', md5_directory + 'all_pages.pdf') # checking for sb transactions if has_sb_schedules: pypdftk.concat([md5_directory + 'all_pages.pdf', md5_directory + 'SB/all_pages.pdf'], md5_directory + 'temp_all_pages.pdf') shutil.move(md5_directory + 'temp_all_pages.pdf', md5_directory + 'all_pages.pdf') os.remove(md5_directory + 'SB/all_pages.pdf') shutil.rmtree(md5_directory + 'SB') else: # no summary, expecting it to be from individual transactions if has_sa_schedules: if os.path.exists(md5_directory + 'all_pages.pdf'): os.remove(md5_directory + 'all_pages.pdf') shutil.move(md5_directory + 'SA/all_pages.pdf', md5_directory + 'all_pages.pdf') else: shutil.move(md5_directory + 'SA/all_pages.pdf', md5_directory + 'all_pages.pdf') shutil.rmtree(md5_directory + 'SA') if has_sb_schedules: if os.path.exists(md5_directory + 'all_pages.pdf'): os.remove(md5_directory + 'all_pages.pdf') shutil.move(md5_directory + 'SB/all_pages.pdf', md5_directory + 'all_pages.pdf') else: shutil.move(md5_directory + 'SB/all_pages.pdf', md5_directory + 'all_pages.pdf') shutil.rmtree(md5_directory + 'SB') # push output file to AWS s3 = boto3.client('s3') s3.upload_file(md5_directory + 'all_pages.pdf', current_app.config['AWS_FECFILE_COMPONENTS_BUCKET_NAME'], md5_directory + 'all_pages.pdf', ExtraArgs={'ContentType': "application/pdf", 'ACL': "public-read"}) response = { # 'file_name': '{}.pdf'.format(json_file_md5), 'pdf_url': current_app.config['PRINT_OUTPUT_FILE_URL'].format(json_file_md5) + 'all_pages.pdf' } # return response if flask.request.method == "POST": envelope = common.get_return_envelope( data=response ) status_code = status.HTTP_201_CREATED return flask.jsonify(**envelope), status_code else: if flask.request.method == "POST": envelope = common.get_return_envelope( 'false', 'JSON file is missing from your request' ) status_code = status.HTTP_400_BAD_REQUEST return flask.jsonify(**envelope), status_code