def test_load_items():
database = Database()
database.delete_item('members', 'testid1')
database.delete_item('members', 'testid2')
columns = database.get_columns('members')
item1 = {x: None for x in columns}
item1['id'] = 'testid1'
item2 = {x: None for x in columns}
item2['id'] = 'testid2'
items = [item1, item2]
database.load_items(items, 'members')
item1_ = database.get_item('members', 'testid1')
assert item1_['id'] == 'testid1'
item2_ = database.get_item('members', 'testid2')
assert item2_['id'] == 'testid2'
database.delete_item('members', 'testid1')
item1_ = database.get_item('members', 'testid1')
assert item1_ == None
database.delete_item('members', 'testid2')
item2_ = database.get_item('members', 'testid2')
assert item2_ == None
class MM2000:
def __init__(self, database=None):
daiquiri.setup(level=logging.INFO)
self.logger = daiquiri.getLogger(__name__)
# Load column mapping configs
self.path = os.path.dirname(os.path.realpath(__file__))
filename = self.path + '/member_columns.yml'
with open(filename, 'r') as f:
self.column_mapping = yaml.safe_load(f)
self.database = Database() if not database else database
self.fake_news = FakeNews(database=self.database)
#####################################
# Methods for loading MM2000 members
#####################################
def load(self, df):
""" Loads the data in to the member database """
self.logger.info('Parsing MM2000 data.')
items = self.parse_mm2000(df)
self.logger.info('Backing up current member table.')
self.database.backup_table('members')
self.logger.info('Truncating current member table.')
self.database.truncate_table('members')
self.logger.info('Loading updated member data.')
for item in items:
self.database.load_item(item, 'members')
self.logger.info('Checking updated columns.')
good_columns = self.check_columns()
if good_columns:
self.logger.info('Generating demo data')
self.fake_news.fake_names()
self.logger.info('Refreshing materialized views.')
self.database.refresh_view('members_view')
self.database.refresh_view('participants')
else:
self.logger.warning('Column mismatch in upload')
self.database.revert_table('members')
return False
return True
def parse_mm2000(self, df):
""" Converts the MM2000 export into a list of rows """
column_mapping = self.column_mapping['MM2000']
items = []
for group in column_mapping:
column_map = column_mapping[group]['columns']
df_group = _group_mm2000(df, column_map)
if 'id_extension' in column_mapping[group]:
id_extension = column_mapping[group]['id_extension']
else:
id_extension = None
for i in df_group.index:
item = dict(df_group.loc[i])
item = _parse_postal_code(item)
item = _check_mm2000_active(item)
# ID extension for children and spouses
# since a family shares the same id
item['household_id'] = item['id']
if id_extension:
item['id'] += id_extension
# Remove invalid birthdates
item = _parse_mm2000_date(item, 'birth_date')
item = _parse_mm2000_date(item, 'membership_date')
# Skip if the member is under the minimum age
# that we keep in the database
too_young = utils.check_age(item['birth_date'], min_age=18)
if too_young:
continue
# Children only have a full name, not separate
# first names and last name
if 'first_name' not in item and item['full_name']:
item['first_name'] = item['full_name'].split()[0]
if 'last_name' not in item and item['full_name']:
item['last_name'] = item['full_name'].split()[0]
if not item['first_name'] or not item['last_name']:
continue
else:
items.append(item)
return items
def check_columns(self):
""" Checks to make sure the columns are the same in the new table """
new_columns = self.database.get_columns('members')
old_columns = self.database.get_columns('members_backup')
for column in new_columns:
if column not in old_columns:
return False
return True
###########################################
# Methods for handling MM2000 resignations
###########################################
def load_resignations(self, df):
"""Loads MM2000 resignation data into the database."""
_validate_resignation_data(df)
# Map the file column names to the databse column names
df = df.rename(columns=self.column_mapping['MM2000 Resignations'])
# Drop any rows where the resignation date is null
df = df.dropna(axis=0, how='any', subset=['resignation_date'])
for i in df.index:
member = dict(df.loc[i])
member = _parse_mm2000_date(member, 'resignation_date')
resignation_date = str(member['resignation_date'])[:10]
# TODO: This logic is specific to TRS because that's how they
# track people who rejoined the congregation. We may have to
# update this if another client uses MM2000
if 'Comment1' in member:
if 'rejoin' in str(member['Comment1']).lower():
resignation_date = None
if 'Comment2' in member:
if 'rejoin' in str(member['Comment2']).lower():
resignation_date = None
if resignation_date:
resignation_date = "'{}'".format(resignation_date)
sql = """
UPDATE {schema}.members
SET resignation_date = {resignation_date}
WHERE (household_id = '{member_id}'
OR id = '{member_id}')
""".format(schema=self.database.schema,
resignation_date=resignation_date,
member_id=member['id'])
self.database.run_query(sql)
reason = _find_resignation_reason(member['resignation_reason'])
sql = """
UPDATE {schema}.members
SET resignation_reason = '{reason}'
WHERE (household_id = '{member_id}'
OR id = '{member_id}')
""".format(schema=self.database.schema,
reason=reason,
member_id=member['id'])
self.database.run_query(sql)
self.database.refresh_views()
def test_get_columns():
database = Database()
columns = database.get_columns('events')
assert len(columns) > 0