folder = van.get_folder(os.environ["VAN_FOLDER_ID"]) # If you're receiving errors when trying to call this method related to permissions, you may want to # reach out to [email protected] to make sure your API key has the correct permissions. saved_list_download = van.download_saved_list(os.environ["VAN_SAVED_LIST_ID"]) # Generate a random sample of VAN IDs from the list saved_list_sample_ids = random.sample( saved_list_download["VanID"], int(os.environ["VAN_SAMPLE_LIST_SIZE"]), ) # Create a new sample list from the sampled IDs that only includes the VanID column saved_list_sample = saved_list_download.cut("VanID").select_rows( lambda row: row["VanID"] in saved_list_sample_ids) logger.info( f"Uploading saved list '{os.environ['VAN_SAMPLE_LIST_NAME']}' with " f"{os.environ['VAN_SAMPLE_LIST_SIZE']} people") # Upload to VAN through an intermediate S3 bucket where we save the data van.upload_saved_list( saved_list_sample, os.environ["VAN_SAMPLE_LIST_NAME"], folder["folderId"], "S3", id_type="vanid", replace=True, bucket=os.environ["S3_BUCKET"], )
if value.strip() != "":
os.environ[name] = value
rs = Redshift()
ak = ActionKit()
# This example involves adding a voterbase_id (the Targetsmart ID) to a user in ActionKit
timestamp = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") # timestamp to be used for log table
loaded = [['id', 'voterbase_id', 'date_updated']] # column names for log table
source_table = 'schema.table' # this is the table with the information I'm pushing to ActionKit
log_table = 'schema.table' # this is where we will log every user id that gets marked with a voterbase_id
logger.info("Running query to get matches...")
query = '''
select distinct id, voterbase_id
from {source_table}
left join {log_table} using (id, voterbase_id)
where voterbase_id is not null and date_updated is null
'''
source_data = rs.query(query)
if source_data.num_rows > 0:
logger.info(f"Will be updating voterbase_id for {source_data.num_rows}...")
for row in source_data:
user = ak.get_user(user_id=row['id'])
user_dict = {"fields": {"vb_voterbase_id": row['voterbase_id']}}
update_user = ak.update_user(user_id=row['id'], **user_dict)
# Setup
for name, value in config_vars.items(
): # sets variables if provided in this script
if value.strip() != "":
os.environ[name] = value
s3 = S3()
rs = Redshift()
# Code
bucket = os.environ['BUCKET']
keys = s3.list_keys(bucket)
files = keys.keys()
if len(keys) == 0:
logger.info("No files to sync today!")
else:
logger.info(f"Pulling {str(len(files))} files down from s3...")
for x in files:
file = s3.get_file(bucket, x)
table = Table.from_csv(file, encoding="ISO-8859-1")
table_name = f"schema.{x.replace('.csv', '')}"
try:
table.to_redshift(table_name, if_exists='truncate')
except:
table.to_redshift(table_name, if_exists='drop')
utilities.files.close_temp_file(file)
from parsons import Redshift, S3, utilities, logger
# Setup
for name, value in config_vars.items(
): # sets variables if provided in this script
if value.strip() != "":
os.environ[name] = value
s3_source = S3(os.environ['AWS_SOURCE_ACCESS_KEY_ID'],
os.environ['AWS_SOURCE_SECRET_ACCESS_KEY'])
s3_destination = S3(os.environ['AWS_DESTINATION_ACCESS_KEY_ID'],
os.environ['AWS_DESTINATION_SECRET_ACCESS_KEY'])
# Let's write some code!
# Get Source Bucket Information
bucket_guide = s3_source.list_buckets()
logger.info(f"We will be getting data from {len(bucket_guide)} buckets...")
# Moving Files from Source s3 Bucket to Destination s3 Bucket
for bucket in bucket_guide:
logger.info(f"Working on files for {bucket}...")
keys = s3_source.list_keys(bucket)
logger.info(f"Found {len(keys)}.")
for key in keys:
temp_file = s3_source.get_file(bucket, key)
s3_destination.put_file(DESTINATION_BUCKET, key, temp_file)
utilities.files.close_temp_file(temp_file)
for k, v in myv_states.items()
}
# Create simple set of states for insertion into SQL
states = "','".join([s for s in myv_keys])
# SQL to pull those needing Activist Code
sql = f"""
SELECT vb_smartvan_id
, vb_vf_source_state
, hash
, activist_code_id
FROM schema.table
WHERE vb_vf_source_state IN ({states})
"""
records = rs.query(sql)
logger.info(f"Applying Activist Codes to {str(records.num_rows)} records...")
# Apply codes segmented by state (different API Keys)
for state, key in myv_keys.items():
state_set = records.select_rows(
lambda row: row.vb_vf_source_state == state)
if len(state_set) > 0:
logger.info(
f"Applying {str(len(state_set))} Activist Codes in {state}...")
for vanid in state_set:
key.toggle_activist_code(row['vb_smartvan_id'],
row['activist_code_id'], 'apply')
editor_email = ''
if not editor_email:
raise ValueError("editor_email is required to enable access to the new Google Sheet")
# Step 2: Specify what contribution data you want from ActBlue
date_range_start = '2022-01-01' # Start of date range to withdraw contribution data (inclusive).
date_range_end = '2022-02-01' # End of date range to withdraw contribution data (exclusive).
csv_type = 'paid_contributions'
# csv_type options:
# 'paid_contributions':
# contains paid, non-refunded contributions to the entity (campaign or organization) you
# created the credential for, during the specified date range
# 'refunded_contributions':
# contributions to your entity that were refunded, during the specified date range
# 'managed_form_contributions':
# contributions made through any form that is managed by your entity, during the specified
# date range - including contributions to other entities via that form if it is a tandem
# form.
# Step 3: Retrieve data from ActBlue and hold it in a Parsons Table.
contribution_data = actblue.get_contributions(csv_type, date_range_start, date_range_end)
# Step 4: Create a spreadsheet on Google Sheets
sheet_id = google_sheets.create_spreadsheet(spreadsheet_name, editor_email=editor_email)
logger.info(f"Created a spreadsheet named {spreadsheet_name}.")
# Step 5: Send data from the Parsons Table to the Google Sheets spreadsheet
google_sheets.append_to_sheet(sheet_id, contribution_data)
logger.info("Completed transfer of data to Google Sheets.")
logger.info(f"Google will email {editor_email} with a link to the new Google Sheet.")