def upload_file(bucket, dry_run, file_path, framework_slug, bucket_category, document_category=None, document_type=None, supplier_name_dict=None): supplier_id = get_supplier_id_from_framework_file_path(file_path) if document_category is None: document_name = get_document_name_from_file_path(file_path) else: document_name = '{0}.{1}'.format(document_category, document_type) if 'signed-framework-agreement' in document_name: print('Signed and countersigned agreement paths now need to be stored in database so can no longer be uploaded ' 'using this script.') return upload_path = get_document_path(framework_slug, supplier_id, bucket_category, document_name) if supplier_name_dict is None: download_filename = None else: supplier_name = supplier_name_dict[supplier_id] download_filename = generate_download_filename( supplier_id, document_name, supplier_name) if not dry_run: with open(file_path) as source_file: bucket.save(upload_path, source_file, acl='private', move_prefix=None, download_filename=download_filename) print(supplier_id) else: print("[Dry-run] UPLOAD: '{}' to '{}'".format(file_path, upload_path))
def upload_to_submissions_bucket(document_name, bucket, bucket_category, framework_slug, supplier_id, file_path, dry_run, supplier_name_dict): # Construct the upload path upload_path = '{0}/{1}/{2}/{3}'.format(framework_slug, bucket_category, supplier_id, document_name) # Download filename, if available if supplier_name_dict is None: download_filename = None else: supplier_name = supplier_name_dict[supplier_id] download_filename = generate_download_filename(supplier_id, document_name, supplier_name) print(download_filename) # Do the actual upload if not dry_run: with open(file_path, 'rb') as source_file: # TODO check if we need download_filename for submissions bucket.save(upload_path, source_file, acl='bucket-owner-full-control', download_filename=download_filename) print(f"Successfully uploaded {upload_path} to S3") else: print("[Dry-run] UPLOAD: '{}' to '{}'".format(file_path, upload_path))
def upload_countersigned_agreement_file(supplier_id, framework_slug): supplier_framework = data_api_client.get_supplier_framework_info( supplier_id, framework_slug)['frameworkInterest'] if not supplier_framework['onFramework'] or supplier_framework[ 'agreementStatus'] in (None, 'draft'): abort(404) agreement_id = supplier_framework['agreementId'] agreements_bucket = s3.S3(current_app.config['DM_AGREEMENTS_BUCKET']) errors = {} if request.files.get('countersigned_agreement'): the_file = request.files['countersigned_agreement'] if not file_is_pdf(the_file): errors['countersigned_agreement'] = 'not_pdf' flash(COUNTERSIGNED_AGREEMENT_NOT_PDF_MESSAGE) if 'countersigned_agreement' not in errors.keys(): supplier_name = supplier_framework.get( 'declaration', {}).get('nameOfOrganisation') if not supplier_name: supplier_name = data_api_client.get_supplier( supplier_id)['suppliers']['name'] if supplier_framework['agreementStatus'] not in [ 'approved', 'countersigned' ]: data_api_client.approve_agreement_for_countersignature( agreement_id, current_user.email_address, current_user.id) path = generate_timestamped_document_upload_path( framework_slug, supplier_id, 'agreements', COUNTERPART_FILENAME) download_filename = generate_download_filename( supplier_id, COUNTERPART_FILENAME, supplier_name) agreements_bucket.save(path, the_file, acl='bucket-owner-full-control', move_prefix=None, download_filename=download_filename) data_api_client.update_framework_agreement( agreement_id, {"countersignedAgreementPath": path}, current_user.email_address) data_api_client.create_audit_event( audit_type=AuditTypes.upload_countersigned_agreement, user=current_user.email_address, object_type='suppliers', object_id=supplier_id, data={'upload_countersigned_agreement': path}) flash(UPLOAD_COUNTERSIGNED_AGREEMENT_MESSAGE) return redirect( url_for('.list_countersigned_agreement_file', supplier_id=supplier_id, framework_slug=framework_slug))
def upload_counterpart_file(bucket, framework_slug, file_path, dry_run, client): supplier_id = get_supplier_id_from_framework_file_path(file_path) supplier_framework = client.get_supplier_framework_info(supplier_id, framework_slug) supplier_framework = supplier_framework['frameworkInterest'] supplier_name = supplier_framework['declaration']['nameOfOrganisation'] download_filename = generate_download_filename(supplier_id, COUNTERPART_FILENAME, supplier_name) upload_path = generate_timestamped_document_upload_path( framework_slug, supplier_id, "agreements", COUNTERPART_FILENAME ) try: if not dry_run: # Upload file with open(file_path) as source_file: bucket.save(upload_path, source_file, acl='private', move_prefix=None, download_filename=download_filename) logger.info("UPLOADED: '{}' to '{}'".format(file_path, upload_path)) # Save filepath to framework agreement client.update_framework_agreement( supplier_framework['agreementId'], {"countersignedAgreementPath": upload_path}, 'upload-counterpart-agreements script run by {}'.format(getpass.getuser()) ) logger.info("countersignedAgreementPath='{}' for agreement ID {}".format( upload_path, supplier_framework['agreementId']) ) else: logger.info("[Dry-run] UPLOAD: '{}' to '{}'".format(file_path, upload_path)) logger.info("[Dry-run] countersignedAgreementPath='{}' for agreement ID {}".format( upload_path, supplier_framework['agreementId']) ) except (OSError, IOError) as e: logger.error("Error reading file '{}': {}".format(file_path, e.message)) except S3ResponseError as e: logger.error("Error uploading '{}' to '{}': {}".format(file_path, upload_path, e.message)) except APIError as e: logger.error("API error setting upload path '{}' on agreement ID {}: {}".format( upload_path, supplier_framework['agreementId'], e.message) )
def upload_file(bucket, dry_run, file_path, framework_slug, bucket_category, document_category=None, document_type=None, supplier_name_dict=None): supplier_id = get_supplier_id_from_framework_file_path(file_path) if document_category is None: document_name = get_document_name_from_file_path(file_path) else: document_name = '{0}.{1}'.format(document_category, document_type) if 'signed-framework-agreement' in document_name: print( 'Signed and countersigned agreement paths now need to be stored in database so can no longer be uploaded ' 'using this script.') return upload_path = get_document_path(framework_slug, supplier_id, bucket_category, document_name) if supplier_name_dict is None: download_filename = None else: supplier_name = supplier_name_dict[supplier_id] download_filename = generate_download_filename(supplier_id, document_name, supplier_name) if not dry_run: with open(file_path, 'rb') as source_file: bucket.save(upload_path, source_file, acl='bucket-owner-full-control', download_filename=download_filename) print(supplier_id) else: print("[Dry-run] UPLOAD: '{}' to '{}'".format(file_path, upload_path))
def test_generate_download_filename(): assert generate_download_filename(584425, 'result-letter.pdf', 'ICNT_Consulting_Ltd') == 'ICNT_Consulting_Ltd-584425-result-letter.pdf' # noqa
def upload_counterpart_file( bucket, framework, file_path, dry_run, data_api_client, dm_notify_client=None, notify_template_id=None, notify_fail_early=True, logger=None, ): if bool(dm_notify_client) != bool(notify_template_id): raise TypeError( "Either specify both dm_notify_client and notify_template_id or neither" ) logger = logger or logging_helpers.getLogger() supplier_id = get_supplier_id_from_framework_file_path(file_path) supplier_framework = data_api_client.get_supplier_framework_info( supplier_id, framework["slug"]) supplier_framework = supplier_framework['frameworkInterest'] supplier_name = ( supplier_framework['declaration']['supplierRegisteredName'] if 'supplierRegisteredName' in supplier_framework['declaration'] else supplier_framework['declaration']['nameOfOrganisation']) download_filename = generate_download_filename(supplier_id, COUNTERPART_FILENAME, supplier_name) email_addresses_to_notify = dm_notify_client and frozenset( chain( (supplier_framework["declaration"]["primaryContactEmail"], ), (user["emailAddress"] for user in data_api_client.find_users_iter( supplier_id=int(supplier_id)) if user["active"]), )) upload_path = generate_timestamped_document_upload_path( framework["slug"], supplier_id, "agreements", COUNTERPART_FILENAME) try: if not dry_run: # Upload file - need to open in binary mode as it's not plain text with open(file_path, 'rb') as source_file: bucket.save(upload_path, source_file, acl='bucket-owner-full-control', download_filename=download_filename) logger.info("UPLOADED: '{}' to '{}'".format( file_path, upload_path)) # Save filepath to framework agreement data_api_client.update_framework_agreement( supplier_framework['agreementId'], {"countersignedAgreementPath": upload_path}, 'upload-counterpart-agreements script run by {}'.format( getpass.getuser())) logger.info( "countersignedAgreementPath='{}' for agreement ID {}".format( upload_path, supplier_framework['agreementId'])) else: logger.info("[Dry-run] UPLOAD: '{}' to '{}'".format( file_path, upload_path)) logger.info( "[Dry-run] countersignedAgreementPath='{}' for agreement ID {}" .format(upload_path, supplier_framework['agreementId'])) failed_send_email_calls = 0 for notify_email in (email_addresses_to_notify or ()): try: if not dry_run: dm_notify_client.send_email( notify_email, notify_template_id, { "framework_slug": framework["slug"], "framework_name": framework["name"], "supplier_name": supplier_name, }, allow_resend=True) logger.debug( f"NOTIFY: sent email to supplier '{supplier_id}' user {hash_string(notify_email)}" ) else: logger.info( f"[Dry-run] Send notify email to supplier '{supplier_id}' user {hash_string(notify_email)}" ) except EmailError as e: logger.error( f"NOTIFY: Error sending email to supplier '{supplier_id}' user {hash_string(notify_email)}" ) if isinstance(e, EmailTemplateError): raise # do not try to continue if notify_fail_early: raise else: failed_send_email_calls += 1 # just catching these exceptions for logging then reraising except (OSError, IOError) as e: logger.error("Error reading file '{}': {}".format( file_path, e.message)) raise except S3ResponseError as e: logger.error("Error uploading '{}' to '{}': {}".format( file_path, upload_path, e.message)) raise except APIError as e: logger.error( "API error setting upload path '{}' on agreement ID {}: {}".format( upload_path, supplier_framework['agreementId'], e.message)) raise if failed_send_email_calls: raise EmailError("{} notify send_emails calls failed".format( failed_send_email_calls))