def main(argv): g.program_filename = os.path.basename(__file__) if g.program_filename[-3:] == '.py': g.program_filename = g.program_filename[:-3] # Load AWS creds which are used for iterating S3 backups and creating download link aws_access_key_id = util.get_ini_setting('aws', 'access_key_id', False) aws_secret_access_key = util.get_ini_setting('aws', 'secret_access_key', False) aws_region_name = util.get_ini_setting('aws', 'region_name', False) aws_s3_bucket_name = util.get_ini_setting('aws', 's3_bucket_name', False) # Find latest backup in 'daily' folder of S3 bucket 'ingomarchurch_website_backups' s3 = boto3.resource('s3', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=aws_region_name) file_items = [item for item in s3.Bucket(aws_s3_bucket_name).objects.all() if item.key[-1] != '/'] newest_sortable_str = '' obj_to_retrieve = None for file_item in file_items: path_sects = file_item.key.split('/') if len(path_sects) == 2: if path_sects[0] == 'daily': filename = path_sects[1] match = re.match('backwpup_[0-9a-f]{6}_(?P<year>[0-9]{4})-(?P<month>[0-9]{2})-(?P<day>[0-9]{2})' + \ '_(?P<hours>[0-9]{2})-(?P<minutes>[0-9]{2})-(?P<seconds>[0-9]{2})\.tar\.gz', filename) if match is not None: sortable_str = match.group('year') + match.group('month') + match.group('day') + \ match.group('hours') + match.group('minutes') + match.group('seconds') if sortable_str > newest_sortable_str: newest_sortable_str = sortable_str obj_to_retrieve = file_item else: message("Unrecognized file in 'daily' backup folder...ignoring: " + file_item.key) else: message('Unrecognized folder or file in website_backups S3 bucket with long path...ignoring: ' + file_item.key) if obj_to_retrieve is not None: # Generate 10-minute download URL s3Client = boto3.client('s3', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=aws_region_name) url = s3Client.generate_presigned_url('get_object', Params = {'Bucket': aws_s3_bucket_name, 'Key': obj_to_retrieve.key}, ExpiresIn = 10 * 60) print url else: message('Error finding latest backup file to retrieve. Aborting!') sys.exit(1) sys.exit(0)
def main(argv): global g parser = argparse.ArgumentParser() parser.add_argument('--output-filename', required=False, help='Output CSV filename. Defaults to ./tmp/pledges_[datetime_stamp].csv') parser.add_argument('--message-output-filename', required=False, help='Filename of message output file. If ' + 'unspecified, defaults to stderr') g.args = parser.parse_args() message_level = util.get_ini_setting('logging', 'level') util.set_logger(message_level, g.args.message_output_filename, os.path.basename(__file__)) ccb_app_username = util.get_ini_setting('ccb', 'app_username', False) ccb_app_password = util.get_ini_setting('ccb', 'app_password', False) ccb_subdomain = util.get_ini_setting('ccb', 'subdomain', False) curr_date_str = datetime.datetime.now().strftime('%m/%d/%Y') pledge_summary_report_info = { "id":"", "type":"pledge_giving_summary", "pledge_type":"family", "date_range":"", "ignore_static_range":"static", "start_date":"01/01/1990", "end_date":curr_date_str, "campus_ids":["1"], "output":"csv" } pledge_summary_request = { 'request': json.dumps(pledge_summary_report_info), 'output': 'export' } pledge_detail_dialog_report_info = { "type":"pledge_giving_detail", "id":"" } pledge_detail_dialog_request = { 'aj': 1, 'ax': 'create_modal', 'request': json.dumps(pledge_detail_dialog_report_info), } pledge_detail_report_info = { 'id':'', 'type': 'pledge_giving_detail', 'transaction_detail_type_id': '{coa_id}', # {coa_id} is substituted at run-time 'print_type': 'family', 'split_child_records': '1', 'show': 'all', 'date_range': '', 'ignore_static_range': 'static', 'start_date': '01/01/1990', 'end_date': curr_date_str, 'campus_ids': ['1'], 'output': 'csv' } pledge_detail_request = { 'request': json.dumps(pledge_detail_report_info), # This is also replaced at run-time 'output': 'export' } with requests.Session() as http_session: util.login(http_session, ccb_subdomain, ccb_app_username, ccb_app_password) # Get list of pledged categories pledge_summary_response = http_session.post('https://' + ccb_subdomain + '.ccbchurch.com/report.php', data=pledge_summary_request) pledge_summary_succeeded = False if pledge_summary_response.status_code == 200: match_pledge_summary_info = re.search('COA Category', pledge_summary_response.text) if match_pledge_summary_info != None: pledge_summary_succeeded = True if not pledge_summary_succeeded: logging.error('Pledge Summary retrieval failure. Aborting!') util.sys_exit(1) csv_reader = csv.reader(StringIO.StringIO(pledge_summary_response.text.encode('ascii', 'ignore'))) header_row = True list_pledge_categories = [] for row in csv_reader: if header_row: assert row[0] == 'COA Category' header_row = False else: list_pledge_categories.append(unicode(row[0])) # Get dictionary of category option IDs report_page = http_session.get('https://' + ccb_subdomain + '.ccbchurch.com/service/report_settings.php', params=pledge_detail_dialog_request) if report_page.status_code == 200: match_report_options = re.search( '<select\s+name=\\\\"transaction_detail_type_id\\\\"\s+id=\\\\"\\\\"\s*>(.*?)<\\\/select>', report_page.text) pledge_categories_str = match_report_options.group(1) else: logging.error('Error retrieving report settings page. Aborting!') util.sys_exit(1) dict_pledge_categories = {} root_str = '' for option_match in re.finditer(r'<option\s+value=\\"([0-9]+)\\"\s*>([^<]*)<\\/option>', pledge_categories_str): if re.match(r' ', option_match.group(2)): dict_pledge_categories[root_str + ' : ' + option_match.group(2)[6:]] = int(option_match.group(1)) else: root_str = option_match.group(2) dict_pledge_categories[root_str] = int(option_match.group(1)) # Loop over each category with pledges and pull back CSV list of pledges for that category output_csv_header = None if g.args.output_filename is not None: output_filename = g.args.output_filename else: output_filename = './tmp/pledges_' + datetime.datetime.now().strftime('%Y%m%d%H%M%S') + '.csv' util.test_write(output_filename) with open(output_filename, 'wb') as csv_output_file: csv_writer = csv.writer(csv_output_file) for pledge_category in list_pledge_categories: logging.info('Retrieving pledges for ' + pledge_category) if pledge_category in dict_pledge_categories: pledge_detail_report_info['transaction_detail_type_id'] = \ str(dict_pledge_categories[pledge_category]) pledge_detail_request['request'] = json.dumps(pledge_detail_report_info) pledge_detail_response = http_session.post('https://' + ccb_subdomain + \ '.ccbchurch.com/report.php', data=pledge_detail_request) pledge_detail_succeeded = False if pledge_detail_response.status_code == 200 and pledge_detail_response.text[:8] == 'Name(s),': pledge_detail_succeeded = True csv_reader = csv.reader(StringIO.StringIO(pledge_detail_response.text.encode('ascii', 'ignore'))) header_row = True for row in csv_reader: if header_row: header_row = False if output_csv_header is None: output_csv_header = ['COA ID', 'COA Category'] + row amount_column_index = output_csv_header.index('Total Pledged') csv_writer.writerow(output_csv_header) else: row = [dict_pledge_categories[pledge_category], pledge_category] + row if row[amount_column_index] != '0': # Ignore non-pledge (contrib-only) rows csv_writer.writerow(row) if not pledge_detail_succeeded: logging.warning('Pledge Detail retrieval failure for category ' + pledge_category) else: logging.warning('Unknown pledge category. ' + pledge_category) logging.info('Pledge details retrieved successfully and written to ' + output_filename) util.sys_exit(0)
def main(argv): global g parser = argparse.ArgumentParser() parser.add_argument( '--output-filename', required=False, help= 'Output ZIP filename. Defaults to ./tmp/<website_name>_[datetime_stamp].zip' ) parser.add_argument('--message-output-filename', required=False, help='Filename of message output file. If ' \ 'unspecified, then messages are written to stderr as well as into the messages_[datetime_stamp].log file ' \ 'that is zipped into the resulting backup file.') parser.add_argument('--post-to-s3', action='store_true', help='If specified, then the created zip file is ' \ 'posted to Amazon AWS S3 bucket (using bucket URL and password in web_backup.ini file)') parser.add_argument('--delete-zip', action='store_true', help='If specified, then the created zip file is ' \ 'deleted after posting to S3') parser.add_argument('--update-and-secure-wp', action='store_true', help='If specified, then ' \ '/root/bin/update_and_secure_wp utility is run to upgrade Wordpress and plugins and redo security flags '\ 'after backup is completed') parser.add_argument('--website-name', required=False, help='Specified website name is mapped to its ' \ 'hosting directory under /var/www and its contents are recursively zipped and if website is WordPress, ' \ 'wp-config.php is interogated and database .sql backup file created and included in ecrypted zip archive ' \ 'which is posted to S3. If no --website-name is specified, then all websites on this server are listed') parser.add_argument( '--retain-temp-directory', action='store_true', help='If specified, the temp directory ' + 'with output from website directory and WordPress database is not deleted' ) parser.add_argument( '--show-backups-to-do', action='store_true', help='If specified, the ONLY thing that is ' + 'done is backup posts and deletions to S3 are calculated and displayed' ) parser.add_argument('--zip-file-password', required=False, help='If provided, overrides password used to encryt ' \ 'zip file that is created that was specified in web_backup.ini') parser.add_argument('--aws-s3-bucket-name', required=False, help='AWS S3 bucket where output backup zip files ' \ 'are stored') parser.add_argument('--notification-emails', required=False, nargs='*', default=argparse.SUPPRESS, help='If specified, list of email addresses that are emailed upon successful upload to AWS S3, along with ' \ 'accessor link to get at the backup zip file (which is encrypted)') g.args = parser.parse_args() g.program_filename = os.path.basename(__file__) if g.program_filename[-3:] == '.py': g.program_filename = g.program_filename[:-3] message_level = util.get_ini_setting('logging', 'level') script_directory = os.path.dirname(os.path.realpath(__file__)) g.temp_directory = tempfile.mkdtemp(prefix='web_backup_') if g.args.message_output_filename is None: g.message_output_filename = g.temp_directory + '/messages_' + \ datetime.datetime.now().strftime('%Y%m%d%H%M%S') + '.log' else: g.message_output_filename = g.args.message_output_filename util.set_logger(message_level, g.message_output_filename, os.path.basename(__file__)) g.websites = util.get_websites() if g.args.website_name is None or g.args.website_name not in g.websites.keys( ): if g.args.website_name is None: print 'NOTE: --website-name of website to backup was not specified.' else: print 'NOTE: Specified website \'' + g.args.website_name + '\' is not a valid website on this server.' print 'Here\'s a list of websites configured on this server.' print util.print_websites(g.websites) util.sys_exit(0) g.website_directory = g.websites[g.args.website_name]['document_root'] # Don't do work that'd just get deleted if not g.args.post_to_s3 and g.args.delete_zip: message_error( 'Does not make sense to create zip file and delete it without posting to AWS S3. Aborting!' ) util.sys_exit(1) # Load AWS creds which are used for checking need for backup and posting backup file g.aws_access_key_id = util.get_ini_setting('aws', 'access_key_id', False) g.aws_secret_access_key = util.get_ini_setting('aws', 'secret_access_key', False) g.aws_region_name = util.get_ini_setting('aws', 'region_name', False) if g.args.aws_s3_bucket_name is not None: g.aws_s3_bucket_name = g.args.aws_s3_bucket_name else: g.aws_s3_bucket_name = util.get_ini_setting('aws', 's3_bucket_name', False) if g.args.zip_file_password is not None: g.zip_file_password = g.args.zip_file_password else: g.zip_file_password = util.get_ini_setting('zip_file', 'password', False) # Call the base directory the name of the website website_name = os.path.basename(g.website_directory) # Start with assumption no backups to do backups_to_do = None # If user specified just to show work to be done (backups to do), calculate, display, and exit if g.args.show_backups_to_do: backups_to_do = get_backups_to_do(website_name) if backups_to_do is None: message_info('Backups in S3 are already up-to-date. Nothing to do') util.sys_exit(0) else: message_info('There are backups/deletions to do') message_info('Backup plan details: ' + str(backups_to_do)) util.sys_exit(0) # See if there are backups to do backups_to_do = get_backups_to_do(website_name) # If we're posting to S3 and deleting the ZIP file, then utility has been run only for purpose of # posting to S3. See if there are posts to be done and exit if not if g.args.post_to_s3 and g.args.delete_zip and backups_to_do is None: message_info( 'Backups in S3 are already up-to-date. Nothing to do. Exiting!') util.sys_exit(0) # Create ZIP file of website files output_filename = g.temp_directory + '/files.zip' os.chdir(g.website_directory) web_files = os.listdir(g.website_directory) if len(web_files) == 0: message_info('No files in directory ' + g.website_directory + '. Nothing to back up. Aborting.') util.sys_exit(1) exec_zip_list = ['/usr/bin/zip', '-r', output_filename, '.'] message_info('Zipping website files directory') FNULL = open(os.devnull, 'w') exit_status = subprocess.call(exec_zip_list, stdout=FNULL) if exit_status == 0: message_info('Successfully zipped web directory to ' + output_filename) else: message_warning('Error running zip. Exit status ' + str(exit_status)) # Create .sql dump file from website's WordPress database (if applicable) wp_config_filename = g.website_directory + '/wp-config.php' if os.path.isfile(wp_config_filename): output_filename = g.temp_directory + '/database.sql' dict_db_info = get_wp_database_defines( wp_config_filename, ['DB_NAME', 'DB_USER', 'DB_PASSWORD', 'DB_HOST']) message_info('Dumping WordPress MySQL database named ' + dict_db_info['DB_NAME']) mysqldump_string = '/bin/mysqldump -h ' + dict_db_info['DB_HOST'] + ' -u ' + dict_db_info['DB_USER'] + \ ' -p' + dict_db_info['DB_PASSWORD'] + ' ' + dict_db_info['DB_NAME'] + ' --add-drop-table -r ' + \ output_filename try: exec_output = subprocess.check_output(mysqldump_string, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: print 'mysqldump exited with error status ' + str( e.returncode) + ' and error: ' + e.output util.sys_exit(1) # Generate final results output zip filename if g.args.output_filename is not None: output_filename = g.args.output_filename elif g.args.delete_zip: # We're deleting it when we're done, so we don't care about its location/name. Grab temp filename tmp_file = tempfile.NamedTemporaryFile(prefix='web_backup_', suffix='.zip', delete=False) output_filename = tmp_file.name tmp_file.close() os.remove(output_filename) message_info('Temp filename used for final results zip output: ' + output_filename) else: output_filename = script_directory + '/tmp/' + website_name + '_' + \ datetime.datetime.now().strftime('%Y%m%d%H%M%S') + '.zip' # Zip together results files to create final encrypted zip file exec_zip_list = [ '/usr/bin/zip', '-P', g.zip_file_password, '-j', '-r', output_filename, g.temp_directory + '/' ] message_info('Zipping results files together') exit_status = subprocess.call(exec_zip_list, stdout=FNULL) if exit_status == 0: message_info('Successfully zipped all results to temporary file ' + output_filename) else: message_error('Error running zip. Exit status ' + str(exit_status)) util.sys_exit(1) # Push ZIP file into appropriate schedule folders (daily, weekly, monthly, etc.) and then delete excess # backups in each folder list_completed_backups = [] if 'notification_emails' in vars(g.args): list_notification_emails = g.args.notification_emails else: list_notification_emails = None if g.args.post_to_s3 and backups_to_do is not None: for folder_name in backups_to_do: if backups_to_do[folder_name]['do_backup']: s3_key = upload_to_s3(website_name, folder_name, output_filename) expiry_days = { 'daily': 1, 'weekly': 7, 'monthly': 31 }[folder_name] expiring_url = gen_s3_expiring_url(s3_key, expiry_days) message_info('Backup URL ' + expiring_url + ' is valid for ' + str(expiry_days) + ' days') list_completed_backups.append( [folder_name, expiring_url, expiry_days]) for item_to_delete in backups_to_do[folder_name][ 'files_to_delete']: delete_from_s3(item_to_delete) if list_notification_emails is not None: send_email_notification(list_completed_backups, list_notification_emails) # If user asked not to retain temp directory, don't delete it! Else, delete it if g.args.retain_temp_directory: message_info('Retained temporary output directory ' + g.temp_directory) else: shutil.rmtree(g.temp_directory) message_info('Temporary output directory deleted') # If user requested generated zip file be deleted, delete it if g.args.delete_zip: os.remove(output_filename) message_info('Output final results zip file deleted') # If its a Wordpress site and user requested, after backup is complete, run /root/bin/update_and_secure_wp utility if 'wordpress_database' in g.websites[ g.args.website_name] and g.args.update_and_secure_wp: message_info( 'Updating and (re)securing Wordpress after backup as requested') try: exec_output = subprocess.check_output( '/root/bin/update_and_secure_wp ' + g.website_directory, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: print '/root/bin/update_and_secure_wp utility exited with error status ' + str(e.returncode) + \ ' and error: ' + e.output util.sys_exit(1) message_info('Done!') util.sys_exit(0)
def main(argv): global g parser = argparse.ArgumentParser() parser.add_argument('--input-filename', required=False, help='Name of input XML file from previous ' + 'group_profiles XML retrieval. If not specified, groups XML data retreived from CCB REST API.') parser.add_argument('--output-groups-filename', required=False, help='Name of CSV output file listing group ' + 'information. Defaults to ./tmp/groups_[datetime_stamp].csv') parser.add_argument('--output-participants-filename', required=False, help='Name of CSV output file listing ' + 'group participant information. Defaults to ./tmp/group_participants_[datetime_stamp].csv') parser.add_argument('--message-output-filename', required=False, help='Filename of message output file. If ' + 'unspecified, defaults to stderr') parser.add_argument('--keep-temp-file', action='store_true', help='If specified, temp file created with XML ' + 'from REST API call is not deleted') g.args = parser.parse_args() message_level = util.get_ini_setting('logging', 'level') util.set_logger(message_level, g.args.message_output_filename, os.path.basename(__file__)) ccb_subdomain = util.get_ini_setting('ccb', 'subdomain', False) ccb_api_username = util.get_ini_setting('ccb', 'api_username', False) ccb_api_password = util.get_ini_setting('ccb', 'api_password', False) # Set groups and participant filenames and test validity if g.args.output_groups_filename is not None: output_groups_filename = g.args.output_groups_filename else: output_groups_filename = './tmp/groups_' + datetime.datetime.now().strftime('%Y%m%d%H%M%S') + '.csv' util.test_write(output_groups_filename) if g.args.output_participants_filename is not None: output_participants_filename = g.args.output_participants_filename else: output_participants_filename = './tmp/group_participants_' + \ datetime.datetime.now().strftime('%Y%m%d%H%M%S') + '.csv' util.test_write(output_participants_filename) if g.args.input_filename is not None: # Pull groups XML from input file specified by user input_filename = g.args.input_filename else: input_filename = util.ccb_rest_xml_to_temp_file(ccb_subdomain, 'group_profiles', ccb_api_username, ccb_api_password) if input_filename is None: logging.error('Could not retrieve group_profiles, so aborting!') util.sys_exit(1) # Properties to peel off each 'group' node in XML list_group_props = [ 'name', 'description', 'campus', 'group_type', 'department', 'area', 'group_capacity', 'meeting_day', 'meeting_time', 'childcare_provided', 'interaction_type', 'membership_type', 'notification', 'listed', 'public_search_listed', 'inactive' ] participant_nodes = [ 'ccb_api/response/groups/group/director', 'ccb_api/response/groups/group/coach', 'ccb_api/response/groups/group/main_leader', 'ccb_api/response/groups/group/leaders/leader', 'ccb_api/response/groups/group/participants/participant' ] path = [] dict_path_ids = {} group_id = None logging.info('Creating groups and group participants output files.') with open(output_groups_filename, 'wb') as csv_output_groups_file: csv_writer_groups = csv.writer(csv_output_groups_file) csv_writer_groups.writerow(['id'] + list_group_props) with open(output_participants_filename, 'wb') as csv_output_participants_file: csv_writer_participants = csv.writer(csv_output_participants_file) csv_writer_participants.writerow(['group_id', 'participant_id', 'participant_type']) for event, elem in ElementTree.iterparse(input_filename, events=('start', 'end')): if event == 'start': path.append(elem.tag) full_path = '/'.join(path) if full_path == 'ccb_api/response/groups/group': current_group_id = elem.attrib['id'] elif event == 'end': if full_path == 'ccb_api/response/groups/group': # Emit 'groups' row props_csv = util.get_elem_id_and_props(elem, list_group_props) csv_writer_groups.writerow(props_csv) elem.clear() # Throw away 'group' node from memory when done processing it elif full_path in participant_nodes: # Emit 'group_participants' row props_csv = [ current_group_id, elem.attrib['id'], elem.tag ] csv_writer_participants.writerow(props_csv) path.pop() full_path = '/'.join(path) logging.info('Groups written to ' + output_groups_filename) logging.info('Group Participants written to ' + output_participants_filename) # If caller didn't specify input filename, then delete the temporary file we retrieved into if g.args.input_filename is None: if g.args.keep_temp_file: logging.info('Temporary downloaded XML retained in file: ' + input_filename) else: os.remove(input_filename) util.sys_exit(0)
def main(argv): global g parser = argparse.ArgumentParser() parser.add_argument('--output-filename', required=False, help='Output CSV filename. Defaults to ./tmp/[datetime_stamp]_pledges.csv') parser.add_argument('--message-output-filename', required=False, help='Filename of message output file. If ' + 'unspecified, defaults to stderr') g.args = parser.parse_args() message_level = util.get_ini_setting('logging', 'level') util.set_logger(message_level, g.args.message_output_filename, os.path.basename(__file__)) ccb_app_username = util.get_ini_setting('ccb', 'app_username', False) ccb_app_password = util.get_ini_setting('ccb', 'app_password', False) ccb_subdomain = util.get_ini_setting('ccb', 'subdomain', False) curr_date_str = datetime.datetime.now().strftime('%m/%d/%Y') individual_detail_report_info = { 'id':'', 'type': 'export_individuals_change_log', 'print_type': 'export_individuals', 'query_id': '', 'campus_ids': ['1'] } individual_detail_request = { 'request': json.dumps(individual_detail_report_info), 'output': 'export' } with requests.Session() as http_session: util.login(http_session, ccb_subdomain, ccb_app_username, ccb_app_password) # Pull back complete CSV containing detail info for every individual in CCB database output_csv_header = None if g.args.output_filename is not None: output_filename = g.args.output_filename else: output_filename = './tmp/individuals_' + datetime.datetime.now().strftime('%Y%m%d%H%M%S') + '.csv' util.test_write(output_filename) with open(output_filename, 'wb') as csv_output_file: csv_writer = csv.writer(csv_output_file) logging.info('Note that it takes CCB a minute or two to pull retrive all individual information') individual_detail_response = http_session.post('https://' + ccb_subdomain + '.ccbchurch.com/report.php', data=individual_detail_request) individual_detail_succeeded = False if individual_detail_response.status_code == 200 and \ individual_detail_response.text[:16] == '"Individual ID",': individual_detail_succeeded = True csv_reader = csv.reader(StringIO.StringIO(individual_detail_response.text.encode('ascii', 'ignore'))) for row in csv_reader: csv_writer.writerow(row) if not individual_detail_succeeded: logging.error('Individual Detail retrieval failed') util.sys_exit(1) else: logging.info('Individual info successfully retrieved into file ' + output_filename) util.sys_exit(0)
def main(argv): global g # Determine which data sets we're backing up g.backup_data_sets_dict = { "individuals": [True, None], "groups": [True, "participants"], "attendance": [True, "events"], "pledges": [True, None], "contributions": [True, None], } backup_data_sets_str = " ".join([x.upper() for x in g.backup_data_sets_dict]) parser = argparse.ArgumentParser() parser.add_argument( "--output-filename", required=False, help="Output ZIP filename. Defaults to ./tmp/ccb_backup_[datetime_stamp].zip", ) parser.add_argument( "--message-output-filename", required=False, help="Filename of message output file. If " + "unspecified, then messages are written to stderr as well as into the messages_[datetime_stamp].log file " + "that is zipped into the resulting backup file.", ) parser.add_argument( "--post-to-s3", action="store_true", help="If specified, then the created zip file is " + "posted to Amazon AWS S3 bucket (using bucket URL and password in ccb_backup.ini file)", ) parser.add_argument( "--delete-zip", action="store_true", help="If specified, then the created zip file is " + "deleted after posting to S3", ) parser.add_argument( "--source-directory", required=False, help="If provided, then get_*.py utilities are not " + "executed to create new output data, but instead files in this specified directory are used " + "to zip and optionally post to AWS S3", ) parser.add_argument( "--retain-temp-directory", action="store_true", help="If specified, the temp directory " + "without output from get_*.py utilities is not deleted", ) parser.add_argument( "--show-backups-to-do", action="store_true", help="If specified, the ONLY thing that is " + "done is backup posts and deletions to S3 are calculated and displayed", ) parser.add_argument( "--all-time", action="store_true", help="Normally, attendance data is only archived for " + "current year (figuring earlier backups covered earlier years). But specifying this flag, collects " "attendance data not just for this year but across all years", ) parser.add_argument( "--backup-data-sets", required=False, nargs="*", default=argparse.SUPPRESS, help="If unspecified, *all* CCB data is backed up. If specified then one or more of the following " "data sets must be specified and only the specified data sets are backed up: " + backup_data_sets_str, ) parser.add_argument( "--zip-file-password", required=False, help="If provided, overrides password used to encryt " "zip file that is created that was specified in ccb_backup.ini", ) parser.add_argument( "--aws-s3-bucket-name", required=False, help="If provided, overrides AWS S3 bucket where " "output backup zip files are stored", ) parser.add_argument( "--notification-emails", required=False, nargs="*", default=argparse.SUPPRESS, help="If specified, list of email addresses that are emailed upon successful upload to AWS S3, along with " "accessor link to get at the backup zip file (which is encrypted)", ) g.args = parser.parse_args() g.program_filename = os.path.basename(__file__) if g.program_filename[-3:] == ".py": g.program_filename = g.program_filename[:-3] message_level = util.get_ini_setting("logging", "level") g.temp_directory = tempfile.mkdtemp(prefix="ccb_backup_") if g.args.message_output_filename is None: g.message_output_filename = ( g.temp_directory + "/messages_" + datetime.datetime.now().strftime("%Y%m%d%H%M%S") + ".log" ) else: g.message_output_filename = g.args.message_output_filename util.set_logger(message_level, g.message_output_filename, os.path.basename(__file__)) # If specified, validate list of backup_data_sets that we're backing up if "backup_data_sets" in vars(g.args): # If specifying individual data sets to backup, start assuming we're backing up none of them for data_set_name in g.backup_data_sets_dict: g.backup_data_sets_dict[data_set_name][0] = False for backup_data_set in g.args.backup_data_sets: backup_data_set_str = backup_data_set.lower() if backup_data_set_str not in g.backup_data_sets_dict: message_error( "Specified --backup-data-sets value '" + backup_data_set + "' must be one of: " + backup_data_sets_str + ". Aborting!" ) sys.exit(1) else: g.backup_data_sets_dict[backup_data_set_str][0] = True # Don't do work that'd just get deleted if not g.args.post_to_s3 and g.args.delete_zip: message_error("Does not make sense to create zip file and delete it without posting to AWS S3. Aborting!") util.sys_exit(1) # Load AWS creds which are used for checking need for backup and posting backup file g.aws_access_key_id = util.get_ini_setting("aws", "access_key_id", False) g.aws_secret_access_key = util.get_ini_setting("aws", "secret_access_key", False) g.aws_region_name = util.get_ini_setting("aws", "region_name", False) if g.args.aws_s3_bucket_name is not None: g.aws_s3_bucket_name = g.args.aws_s3_bucket_name else: g.aws_s3_bucket_name = util.get_ini_setting("aws", "s3_bucket_name", False) if g.args.zip_file_password is not None: g.zip_file_password = g.args.zip_file_password else: g.zip_file_password = util.get_ini_setting("zip_file", "password", False) # Start with assumption no backups to do backups_to_do = None # If user specified just to show work to be done (backups to do), calculate, display, and exit if g.args.show_backups_to_do: backups_to_do = get_backups_to_do() if backups_to_do is None: message_info("Backups in S3 are already up-to-date. Nothing to do") util.sys_exit(0) else: message_info("There are backups/deletions to do") message_info("Backup plan details: " + str(backups_to_do)) util.sys_exit(0) # See if there are backups to do backups_to_do = get_backups_to_do() # If we're posting to S3 and deleting the ZIP file, then utility has been run only for purpose of # posting to S3. See if there are posts to be done and exit if not if g.args.post_to_s3 and g.args.delete_zip and backups_to_do is None: message_info("Backups in S3 are already up-to-date. Nothing to do. Exiting!") util.sys_exit(0) # If user specified a directory with set of already-created get_*.py utilities output files to use, then # do not run get_*.py data collection utilities, just use that if g.args.source_directory is not None: g.temp_directory = g.args.source_directory else: # Run get_XXX.py utilities into datetime_stamped CSV output files and messages_output.log output in # temp directory g.run_util_errors = [] for data_set_name in g.backup_data_sets_dict: if g.backup_data_sets_dict[data_set_name][0]: run_util(data_set_name, g.backup_data_sets_dict[data_set_name][1]) message_info("Finished all data collection") # Create output ZIP file if g.args.output_filename is not None: output_filename = g.args.output_filename elif g.args.delete_zip: # We're deleting it when we're done, so we don't care about its location/name. Grab temp filename tmp_file = tempfile.NamedTemporaryFile(prefix="ccb_backup_", suffix=".zip", delete=False) output_filename = tmp_file.name tmp_file.close() os.remove(output_filename) print "Temp filename: " + output_filename else: output_filename = "./tmp/ccb_backup_" + datetime.datetime.now().strftime("%Y%m%d%H%M%S") + ".zip" exec_zip_list = ["/usr/bin/zip", "-P", g.zip_file_password, "-j", "-r", output_filename, g.temp_directory + "/"] message_info("Zipping data collection results files") exit_status = subprocess.call(exec_zip_list) if exit_status == 0: message_info("Successfully zipped get_*.py utilities output and messages log to " + output_filename) else: message_warning("Error running zip. Exit status " + str(exit_status)) # Push ZIP file into appropriate schedule folders (daily, weekly, monthly, etc.) and then delete excess # backups in each folder list_completed_backups = [] if "notification_emails" in vars(g.args): list_notification_emails = g.args.notification_emails else: list_notification_emails = None if backups_to_do is not None: for folder_name in backups_to_do: if backups_to_do[folder_name]["do_backup"]: s3_key = upload_to_s3(folder_name, output_filename) expiry_days = {"daily": 1, "weekly": 7, "monthly": 31}[folder_name] expiring_url = gen_s3_expiring_url(s3_key, expiry_days) message_info("Backup URL " + expiring_url + " is valid for " + str(expiry_days) + " days") list_completed_backups.append([folder_name, expiring_url, expiry_days]) for item_to_delete in backups_to_do[folder_name]["files_to_delete"]: delete_from_s3(item_to_delete) if list_notification_emails is not None: send_email_notification(list_completed_backups, list_notification_emails) # If user specified the source directory, don't delete it! And if user asked not to retain temp directory, # don't delete it! if g.args.source_directory is None: if g.args.retain_temp_directory: message_info("Retained temporary output directory " + g.temp_directory) else: shutil.rmtree(g.temp_directory) message_info("Temporary output directory deleted") util.sys_exit(0)
def main(argv): parser = argparse.ArgumentParser() parser.add_argument('--from-website-backup-file', required=False, help='Input ZIP filename of a website ' 'backup file.') parser.add_argument('--from-s3-website-name', required=False, help='Name of website with backup archive in S3.') parser.add_argument('--to-website-name', required=False, help='Name of local website to restore into.') parser.add_argument('--message-output-filename', required=False, help='Filename of message output file. If ' \ 'unspecified, then messages are written to stderr as well as into the messages_[datetime_stamp].log file ' \ 'that is zipped into the resulting backup file.') parser.add_argument('--zip-file-password', required=False, help='If provided, overrides password used to encryt ' \ 'zip file that is created that was specified in web_backup.ini') parser.add_argument('--aws-s3-bucket-name', required=False, help='AWS S3 bucket where output backup zip files ' \ 'are stored') parser.add_argument('--overwrite-files', action='store_true', help='If specified, if there are existing files ' \ 'in the target website directory, they are overwritten') parser.add_argument('--overwrite-database', action='store_true', help='If specified, if there is an existing ' \ 'database with same name as that being restored, it is dropped before replacement created in its place') parser.add_argument('--wp-user', required=False, help='If specified, if the restored site is a Wordpress ' \ 'site, this is the database user used by Wordpress to access the site\'s database') parser.add_argument('--wp-user-password', required=False, help='If specified, if the restored site is a ' \ 'Wordpress site, and --wp-user is specified, then this password is used when creating specified wp-user if ' \ 'that user doesn\'t exist in MySQL') g.args = parser.parse_args() g.program_filename = os.path.basename(__file__) if g.program_filename[-3:] == '.py': g.program_filename = g.program_filename[:-3] message_level = util.get_ini_setting('logging', 'level') if g.args.message_output_filename is not None: g.message_output_filename = g.args.message_output_filename util.set_logger(message_level, g.message_output_filename, os.path.basename(__file__)) g.websites = util.get_websites() if g.args.to_website_name is None or g.args.to_website_name not in g.websites.keys( ): if g.args.to_website_name is None: print 'NOTE: --to-website-name of website to restore into was not specified.' else: print 'NOTE: Specified website \'' + g.args.to_website_name + '\' is not a valid website on this server.' print 'Here\'s a list of websites configured on this server.' print util.print_websites(g.websites) sys.exit(0) if g.args.wp_user is None and g.args.wp_user_password is not None: message_error('You cannot specify new wp_user\'s password without specifying the new wp_user to ' \ 'create with that password.') sys.exit(1) if g.args.from_website_backup_file is None and g.args.from_s3_website_name is None: message_error('Must either specify a local backup ZIP file to restore from or an S3 bucket to grab latest ' \ 'backup from.') sys.exit(1) if g.args.from_website_backup_file is None: # Load AWS creds which are used for iterating S3 backups and creating download link aws_access_key_id = util.get_ini_setting('aws', 'access_key_id', False) aws_secret_access_key = util.get_ini_setting('aws', 'secret_access_key', False) aws_region_name = util.get_ini_setting('aws', 'region_name', False) aws_s3_bucket_name = util.get_ini_setting('aws', 's3_bucket_name', False) # Find latest backup in 'daily' folder of S3 bucket 'ingomarchurch_website_backups' s3 = boto3.resource('s3', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=aws_region_name) file_items = [ item for item in s3.Bucket(aws_s3_bucket_name).objects.all() if item.key[-1] != '/' ] newest_sortable_str = '' obj_to_retrieve = None for file_item in file_items: path_sects = file_item.key.split('/') if len(path_sects) == 3: if path_sects[0] == g.args.from_s3_website_name: if path_sects[1] == 'daily': filename = path_sects[2] match = re.match('(?P<year>[0-9]{4})(?P<month>[0-9]{2})(?P<day>[0-9]{2})' + \ '(?P<hours>[0-9]{2})(?P<minutes>[0-9]{2})(?P<seconds>[0-9]{2})\.zip', filename) if match is not None: sortable_str = match.group('year') + match.group('month') + match.group('day') + \ match.group('hours') + match.group('minutes') + match.group('seconds') if sortable_str > newest_sortable_str: newest_sortable_str = sortable_str obj_to_retrieve = file_item else: message( "Unrecognized file in 'daily' backup folder...ignoring: " + file_item.key) else: message( 'Non-matching folder or file in website_backups S3 bucket...ignoring: ' + file_item.key) else: message( 'Unrecognized folder or file in website_backups S3 bucket with long path...ignoring: ' + file_item.key) if obj_to_retrieve is not None: # Generate 10-minute download URL s3Client = boto3.client( 's3', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=aws_region_name) url = s3Client.generate_presigned_url('get_object', Params={ 'Bucket': aws_s3_bucket_name, 'Key': obj_to_retrieve.key }, ExpiresIn=10 * 60) else: message('Error finding latest backup file to retrieve. Aborting!') sys.exit(1) backup_zip_file = tempfile.NamedTemporaryFile(prefix='web_restore_', suffix='.zip', delete=False) backup_zip_filename = backup_zip_file.name backup_zip_file.close() os.remove(backup_zip_filename) urllib.urlretrieve(url, backup_zip_filename) else: if os.path.exists(g.args.from_website_backup_file): backup_zip_filename = g.args.from_website_backup_file else: message_error('Specified website backup file does not exist: ' + g.args.from_website_backup_file) sys.exit(1) if g.args.zip_file_password is not None: zip_file_password = g.args.zip_file_password else: zip_file_password = util.get_ini_setting('zip_file', 'password', False) temp_directory = tempfile.mkdtemp(prefix='web_restore_') exec_zip_list = [ '/usr/bin/unzip', '-P', zip_file_password, backup_zip_filename, '-d', temp_directory ] message_info('Unzipping backup file container into ' + temp_directory) FNULL = open(os.devnull, 'w') exit_status = subprocess.call(exec_zip_list, stdout=FNULL) website_root = util.get_ini_setting('website', 'root_directory') website_dir = website_root + '/' + g.args.to_website_name if not os.path.isdir(website_dir): message_error(website_dir + ' is not a directory.') sys.exit(1) # Ensure target directory is empty before restoring files into it existing_file_list = os.listdir(website_dir) if len(existing_file_list) != 0: if not g.args.overwrite_files: message_error( website_dir + ' is not empty and --overwrite-files was not specified. Aborting...' ) sys.exit(1) else: message_info('Directory ' + website_dir + ' is not empty. Cleaning first.') for the_file in existing_file_list: file_path = os.path.join(website_dir, the_file) if os.path.isfile(file_path): os.unlink(file_path) elif os.path.isdir(file_path): shutil.rmtree(file_path) # Restore website files exec_zip_list = [ '/usr/bin/unzip', temp_directory + '/files.zip', '-d', website_dir ] message_info('Unzipping backed up website files to ' + website_dir) FNULL = open(os.devnull, 'w') exit_status = subprocess.call(exec_zip_list, stdout=FNULL) # Is there a Wordpress database in the backup for us to restore? if os.path.isfile(temp_directory + '/database.sql'): db_user = util.get_ini_setting('database', 'user', False) db_password = util.get_ini_setting('database', 'password', False) output_lines = subprocess.check_output("/bin/mysql -u " + db_user + " -p" + db_password + \ " -e 'show databases;' 2>/dev/null | /bin/grep wp_", shell=True) output_lines_list = [ elem for elem in output_lines.split("\n") if elem != "" ] db_name = 'wp_' + g.args.to_website_name if db_name in output_lines_list: if not g.args.overwrite_database: message_error('Database ' + db_name + ' already exists and --overwrite_database was ' \ 'not specified. Aborting...') sys.exit(1) # Recreate database from backup's database.sql file wp_user = '******' wp_user_password = None if g.args.wp_user is not None: wp_user = g.args.wp_user if g.args.wp_user is not None and g.args.wp_user_password is not None: wp_user_password = g.args.wp_user_password wrapper_sql_file = create_wrapper_sql_file(db_name, wp_user, wp_user_password, temp_directory) output_lines = subprocess.check_output("/bin/mysql -u " + db_user + " -p" + db_password + \ " < " + temp_directory + "/restore.sql", shell=True) # Update wp-config.php file wp_config_filename = website_root + '/' + g.args.to_website_name + '/wp-config.php' if not os.path.isfile(wp_config_filename): message_error('Wordpress config file ' + wp_config_filename + ' does not exist.') sys.exit(1) with open(wp_config_filename, 'r') as f_in: with open(wp_config_filename + 'x', 'w') as f_out: in_line = f_in.readline() skipping_lines = False while in_line: out_line = None m = re.match( '\s*define\(\s*\'DB_NAME\'*,\s*\'(?P<db_name>[^\']+)\'\s*\);\s*', in_line) if m is not None: curr_db_name = m.group('db_name') new_db_name = 'wp_' + g.args.to_website_name if curr_db_name != new_db_name: out_line = 'define(\'DB_NAME\', \'' + new_db_name + '\');\n' m = re.match( '\s*define\(\s*\'DB_USER\'*,\s*\'(?P<db_user>[^\']+)\'\s*\);\s*', in_line) if m is not None and g.args.wp_user is not None: curr_db_user = m.group('db_user') new_db_user = g.args.wp_user if curr_db_user != new_db_user: out_line = 'define(\'DB_USER\', \'' + new_db_user + '\');\n' m = re.match( '\s*define\(\s*\'DB_PASSWORD\'*,\s*\'(?P<db_password>[^\']+)\'\s*\);\s*', in_line) if m is not None and g.args.wp_user is not None and g.args.wp_user_password is not None: curr_db_password = m.group('db_password') new_db_password = g.args.wp_user_password if curr_db_password != new_db_password: out_line = 'define(\'DB_PASSWORD\', \'' + new_db_password + '\');\n' m = re.match('\s*define\(\s*\'AUTH_KEY\'*', in_line) if m is not None: skipping_lines = True m = re.match('\s*define\(\s*\'NONCE_SALT\'*', in_line) if m is not None: skipping_lines = False send_new_random_salt(f_out) in_line = f_in.readline() next if not skipping_lines: if out_line is None: f_out.write(in_line) else: f_out.write(out_line) in_line = f_in.readline() os.rename(wp_config_filename + 'x', wp_config_filename) # Rename Wordpress URL in database output_lines = subprocess.check_output('mysql -u ' + db_user + ' -p' + db_password + \ ' -e "use wp_' + g.args.to_website_name + ';select option_value from wp_options ' \ 'where option_name = \'siteurl\';"', shell=True) output_lines_list = [ elem for elem in output_lines.split("\n") if elem != "" ] current_full_domain = None for line in output_lines_list: m = re.match('[\s\|]*https://(?P<full_domain>[a-z0-9\.]+)[\s\|]*', line) if m is not None: current_full_domain = m.group('full_domain') new_full_domain = g.websites[g.args.to_website_name]['server_name'] if current_full_domain is not None and current_full_domain != new_full_domain: message_info('Renaming from https://' + current_full_domain + ' to https://' + new_full_domain + \ ' in Wordpress database') output_lines = subprocess.check_output('/usr/local/bin/wp --path=/var/www/' + g.args.to_website_name + \ ' search-replace "https://' + current_full_domain + '" "https://' + new_full_domain + '" ' \ '--skip-columns=guid', shell=True) else: message_info('No need to rename from https://' + current_full_domain + ' to https://' + \ new_full_domain + ' in Wordpress database...skipping') # Update and (re)secure Wordpress after a restore message_info('Updating and (re)securing Wordpress') try: exec_output = subprocess.check_output( '/root/bin/update_and_secure_wp ' + website_dir, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: print '/root/bin/update_and_secure_wp utility exited with error status ' + str(e.returncode) + \ ' and error: ' + e.output sys.exit(1) # Cleanup shutil.rmtree(temp_directory) message_info('Temporary output directory deleted') print 'NOTE: If you web_restore\'d a WordPress installation with WordFence installed, you may need to hand edit' print ' .htaccess to modify auto_prepend_file to point at proper wordfence-waf.php in restored' print ' installation. Also, interfacing with services like Cloudflare, Google Maps, Google Analytics,' print ' NewRelic, Pingdom, etc., may need to be (re)configured to properly support this site.' print print 'Done!' sys.exit(0)
def main(argv): global g parser = argparse.ArgumentParser() parser.add_argument('--input-events-filename', required=False, help='Name of input CSV file from previous ' + 'event occurrences retrieval. If not specified, event list CSV data is retrieved from CCB UI.') parser.add_argument('--output-events-filename', required=False, help='Name of CSV output file listing event ' + 'information. Defaults to ./tmp/events_[datetime_stamp].csv') parser.add_argument('--output-attendance-filename', required=False, help='Name of CSV output file listing ' + 'attendance information. Defaults to ./tmp/attendance_[datetime_stamp].csv') parser.add_argument('--message-output-filename', required=False, help='Filename of message output file. If ' + 'unspecified, defaults to stderr') parser.add_argument('--keep-temp-file', action='store_true', help='If specified, temp event occurrences CSV ' + \ 'file created with CSV data pulled from CCB UI (event list report) is not deleted so it can be used ' + \ 'in subsequent runs') parser.add_argument('--all-time', action='store_true', help='Normally, attendance data is only archived for ' + \ 'current year (figuring earlier backups covered earlier years). But setting this flag, collects ' \ 'attendance data note just for this year but across all years') g.args = parser.parse_args() message_level = util.get_ini_setting('logging', 'level') util.set_logger(message_level, g.args.message_output_filename, os.path.basename(__file__)) g.ccb_subdomain = util.get_ini_setting('ccb', 'subdomain', False) ccb_app_username = util.get_ini_setting('ccb', 'app_username', False) ccb_app_password = util.get_ini_setting('ccb', 'app_password', False) g.ccb_api_username = util.get_ini_setting('ccb', 'api_username', False) g.ccb_api_password = util.get_ini_setting('ccb', 'api_password', False) datetime_now = datetime.datetime.now() curr_date_str = datetime_now.strftime('%m/%d/%Y') if g.args.all_time: start_date_str = '01/01/1990' else: start_date_str = '01/01/' + datetime_now.strftime('%Y') logging.info('Gathering attendance data between ' + start_date_str + ' and ' + curr_date_str) event_list_info = { "id":"", "type":"event_list", "date_range":"", "ignore_static_range":"static", "start_date":start_date_str, "end_date":curr_date_str, "additional_event_types":["","non_church_wide_events","filter_off"], "campus_ids":["1"], "output":"csv" } event_list_request = { 'request': json.dumps(event_list_info), 'output': 'export' } # Set events and attendance filenames and test validity if g.args.output_events_filename is not None: output_events_filename = g.args.output_events_filename else: output_events_filename = './tmp/events_' + datetime.datetime.now().strftime('%Y%m%d%H%M%S') + '.csv' util.test_write(output_events_filename) if g.args.output_attendance_filename is not None: output_attendance_filename = g.args.output_attendance_filename else: output_attendance_filename = './tmp/attendance_' + \ datetime.datetime.now().strftime('%Y%m%d%H%M%S') + '.csv' util.test_write(output_attendance_filename) input_filename = util.ccb_rest_xml_to_temp_file(g.ccb_subdomain, 'event_profiles', g.ccb_api_username, g.ccb_api_password) if input_filename is None: logging.error('CCB REST API call for event_profiles failed. Aborting!') util.sys_exit(1) # Properties to peel off each 'event' node in XML list_event_props = [ 'name', 'description', 'leader_notes', 'start_datetime', 'end_datetime', 'timezone', 'recurrence_description', 'approval_status', 'listed', 'public_calendar_listed' ] # Also collect event_id, group_id, organizer_id path = [] dict_list_event_names = defaultdict(list) with open(output_events_filename, 'wb') as csv_output_events_file: csv_writer_events = csv.writer(csv_output_events_file) csv_writer_events.writerow(['event_id'] + list_event_props + ['group_id', 'organizer_id']) # Write header row for event, elem in ElementTree.iterparse(input_filename, events=('start', 'end')): if event == 'start': path.append(elem.tag) full_path = '/'.join(path) if full_path == 'ccb_api/response/events/event': current_event_id = elem.attrib['id'] elif event == 'end': if full_path == 'ccb_api/response/events/event': # Emit 'events' row props_csv = util.get_elem_id_and_props(elem, list_event_props) event_id = props_csv[0] # get_elem_id_and_props() puts 'id' prop at index 0 name = props_csv[1] # Cheating here...we know 'name' prop is index 1 dict_list_event_names[name].append(event_id) props_csv.append(current_group_id) props_csv.append(current_organizer_id) csv_writer_events.writerow(props_csv) elem.clear() # Throw away 'event' node from memory when done processing it elif full_path == 'ccb_api/response/events/event/group': current_group_id = elem.attrib['id'] elif full_path == 'ccb_api/response/events/event/organizer': current_organizer_id = elem.attrib['id'] path.pop() full_path = '/'.join(path) if g.args.input_events_filename is not None: # Pull calendared events CSV from file input_filename = g.args.input_events_filename else: # Create UI user session to pull list of calendared events logging.info('Logging in to UI session') with requests.Session() as http_session: util.login(http_session, g.ccb_subdomain, ccb_app_username, ccb_app_password) # Get list of all scheduled events logging.info('Retrieving list of all scheduled events. This might take a couple minutes!') event_list_response = http_session.post('https://' + g.ccb_subdomain + '.ccbchurch.com/report.php', data=event_list_request) event_list_succeeded = False if event_list_response.status_code == 200: event_list_response.raw.decode_content = True with tempfile.NamedTemporaryFile(delete=False) as temp: input_filename = temp.name first_chunk = True for chunk in event_list_response.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks if first_chunk: if chunk[:13] != '"Event Name",': logging.error('Mis-formed calendared events CSV returned. Aborting!') util.sys_exit(1) first_chunk = False temp.write(chunk) temp.flush() with open(input_filename, 'rb') as csvfile: csv_reader = csv.reader(csvfile) with open(output_attendance_filename, 'wb') as csv_output_file: csv_writer = csv.writer(csv_output_file) csv_writer.writerow(['event_id', 'event_occurrence', 'individual_id', 'count']) header_row = True for row in csv_reader: if header_row: header_row = False output_csv_header = row event_name_column_index = row.index('Event Name') attendance_column_index = row.index('Actual Attendance') date_column_index = row.index('Date') start_time_column_index = row.index('Start Time') else: # Retrieve attendees for events which have non-zero number of attendees if row[attendance_column_index] != '0': if row[event_name_column_index] in dict_list_event_names: retrieve_attendance(csv_writer, dict_list_event_names[row[event_name_column_index]], row[date_column_index], row[start_time_column_index], row[attendance_column_index]) else: logging.warning("Unrecognized event name '" + row[event_name_column_index] + "'") # If caller didn't specify input filename, then delete the temporary file we retrieved into if g.args.input_events_filename is None: if g.args.keep_temp_file: logging.info('Temporary downloaded calendared events CSV retained in file: ' + input_filename) else: os.remove(input_filename) logging.info('Event profile data written to ' + output_events_filename) logging.info('Attendance data written to ' + output_attendance_filename) util.sys_exit(0)