def save_csv_file(file_data, file_name, metadata): """Given the campaign model use its ID to name the file and save the photo to S3. :param file_data: CSV stream data to save to S3. :param file_name: The file name. :param metadata: File metadata for WebStorage. """ # Get the file_type: file_storage.content_type is something like 'image/png'. WebStorage.save(file_name, file_data.getvalue(), metadata)
def save_image(file_name, file_storage): """Save an image to AWS S3. :param file_name: Something like 2.jpeg :param file_storage: The file storage object :return: """ try: file_storage.seek(0) WebStorage.save(file_name, file_storage.read(), ('Campaign', file_name)) except BotoClientError as error: raise error
def process_paypal_etl(enacted_by_agent_id, reader_list, file_storage): """Handle the logic for PayPal ETL. :param enacted_by_agent_id: The administrative user ID. :param reader_list: The CSV validated and converted to a list of ordered dictionaries. :param file_storage_name: The file storage name. :return: """ data_from_models = get_data_from_models() ids = { 'transaction': data_from_models['transaction_ids'], 'unresolved_transaction': data_from_models['unresolved_transaction_ids'] } agent_emails = get_agent_emails() # Using for SQLAlchemy bulk_save_objects(). bulk_objects = { 'transaction': [], 'caged_donor': [], 'unresolved_transaction': [] } # Start the loop. for row in reader_list: if row['transaction_id'] in data_from_models['transaction_ids']: # This transaction already exists in our database. continue transaction_type = row['type'] if transaction_type in VALID_GIFT_TRANSACTION_TYPES: valid_paypal_transaction(row, enacted_by_agent_id, agent_emails, ids, bulk_objects) elif transaction_type in REFUND_PAYPAL_TRANSACTION_TYPES: refund_paypal_transaction(row, enacted_by_agent_id, ids, bulk_objects) elif transaction_type in DISPUTE_PAYPAL_TRANSACTION_TYPES: # Handle the PayPal disputes. dispute_paypal_transaction(row, enacted_by_agent_id, ids, bulk_objects) elif transaction_type in USELESS_PAYPAL_TRANSACTION_TYPES: # Culling extraneous transaction types from PayPal. continue else: # CSV file might come with some types we don't know how to process yet. bulk_objects['unresolved_transaction'] += filter( None, [ generate_unresolved_transaction( row, data_from_models['unresolved_transaction_ids'], enacted_by_agent_id) ]) # Store into PayPal_ETL table. file_storage_name = file_storage.filename paypal_etl = PaypalETLModel() paypal_etl.enacted_by_agent_id = enacted_by_agent_id file_info = file_storage_name.split('.') file_date = datetime.utcnow() file_name = '.'.join(file_info[:-1]) + '_' + file_date.strftime( '%Y-%m-%d %H:%M:%S') + '.' + file_info[-1] paypal_etl.file_name = file_name paypal_etl.date_in_utc = file_date database.session.add(paypal_etl) # Bulk save various objects. database.session.bulk_save_objects(bulk_objects['transaction']) database.session.bulk_save_objects(bulk_objects['caged_donor']) database.session.bulk_save_objects(bulk_objects['unresolved_transaction']) # Commit to the database. try: database.session.commit() except SQLAlchemyError: database.session.rollback() raise PayPalETLOnCommitError else: # Store file on AWS S3 and return link. file_storage.seek(0) WebStorage.init_storage(current_app, current_app.config.get('AWS_CSV_FILES_BUCKET'), current_app.config.get('AWS_CSV_FILES_PATH')) metadata = ('Paypal ETL', file_name) WebStorage.save(file_name, file_storage.read(), metadata) return True
def get_cron_for_csv(): """A function to be called as a cron job to retrieve a full diump of the donate database to a CSV.""" # Open the stream for CSV and write the header. output = io.BytesIO() output.write(','.join(HEADER).encode()) output.write('\n'.encode()) logging.info('') logging.info('1. Open the production DB connection.') # Handle vault tokens. vault = get_vault_data(app.config['VAULT_URL'], app.config['VAULT_TOKEN'], app.config['VAULT_SECRET']) # Create the database connection. dump_conn = pymysql.connect(host=app.config['DUMP_SQLALCHEMY_HOST'], port=int(app.config['DUMP_SQLALCHEMY_PORT']), user=vault['data']['username'], passwd=vault['data']['password'], db=app.config['DUMP_SQLALCHEMY_DB']) logging.info('2. Get the data.') # Get the MySQL query to extract data from the database. sql_query = query_transactions_for_csv() # Get the cursor and perform query, with the MySQL equivalent of nolock on the database. dump_cursor = dump_conn.cursor() with dump_cursor as cursor: cursor.execute( 'SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;') cursor.execute(sql_query) rows = list(cursor) cursor.execute( 'SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ;') logging.info('3. Write the data.') # Write the query data to the output stream. for row in rows: output.write(','.join(map(str, row)).encode()) output.write('\n'.encode()) logging.info('4. Save the data.') # Save the data to AS S3 and get the URL. metadata = ('Transaction Updater', FILE_NAME) WebStorage.save(FILE_NAME, output.getvalue(), metadata) url = WebStorage.generate_presigned_url( app.config['AWS_CSV_FILES_BUCKET'], app.config['AWS_CSV_FILES_PATH'] + FILE_NAME) # Send a notification email to the group. email = app.config['STATISTICS_GROUP_EMAIL'] data = {'email': email, 'urls': url} ultsys_email_api_key = app.config['ULTSYS_EMAIL_API_KEY'] ultsys_email_url = app.config['ULTSYS_EMAIL_URL'] headers = { 'content-type': 'application/json', 'X-Temporary-Service-Auth': ultsys_email_api_key } requests.post(ultsys_email_url, params=data, headers=headers) logging.info(url)