def _main(osm, auth, sections):
group = Group(osm, auth, MAPPING.keys(), None)
for section in sections:
assert section in group.SECTIONIDS.keys(), \
"section must be in {!r}.".format(group.SECTIONIDS.keys())
contacts = []
for section in sections:
section_contacts = [member2contacts(member, section) for
member in group.section_all_members(section)]
# flatten list of lists.
contacts += list(itertools.chain(*section_contacts))
# Remove blank emails
contacts = [contact for contact in contacts if contact[2].strip() != "" ]
# remove duplicates
by_email = {contact[2]: contact for contact in contacts}
contacts = list(by_email.values())
w = csv_writer(sys.stdout)
w.writerows(contacts)
def member_badges(osm, auth, firstname, lastname, csv=False, no_headers=False, term=None):
group = Group(osm, auth, MAPPING.keys(), term)
members = group.find_by_name(firstname, lastname)
# member = members[-1]
rows = []
for member in members:
for section_type in ('beavers', 'cubs', 'scouts'):
try:
badges = member.get_badges(section_type=section_type)
if badges is not None:
for badge in [_ for _ in badges if _['awarded'] == '1']:
rows.append([member['date_of_birth'], member['last_name'],
member['age'], section_type, member._section['sectionname'],
badge['badge'],
datetime.date.fromtimestamp(int(badge['awarded_date'])).isoformat()])
except:
import traceback
traceback.print_exc()
pass
headers = ["DOB", "Last Name", "Age", "Section Type", "Section Name", "Badge"]
if csv:
w = csv_writer(sys.stdout)
if not no_headers:
w.writerow(list(headers))
w.writerows(rows)
else:
if not no_headers:
print(tabulate.tabulate(rows, headers=headers))
else:
print(tabulate.tabulate(rows, tablefmt="plain"))
def events_info(osm, auth, sections, event, term=None):
group = Group(osm, auth, MAPPING.keys(), term)
for section in sections:
ev = group._sections.sections[
Group.SECTIONIDS[section]].events.get_by_name(event)
print(",".join([ev[_] for _ in ['name', 'startdate', 'enddate', 'location']]))
def payments(osm, auth, sections, start, end):
group = Group(osm, auth, MAPPING.keys(), None)
for section in sections:
osm_section = group._sections.sections[Group.SECTIONIDS[section]]
payments = osm_section.get_payments(start, end)
print(payments.content.decode())
def _main(osm, auth):
#test_section = '15797'
group = Group(osm, auth, MAPPING.keys())
if group.missing_adult_references():
log.warn("Missing adult references {!r}".format(
group.missing_adult_references()))
def movers_list(osm, auth, sections, age=None, term=None,
csv=False, no_headers=False):
group = Group(osm, auth, MAPPING.keys(), term)
rows = []
for section in sections:
section_ = group._sections.sections[Group.SECTIONIDS[section]]
headers = ['firstname', 'lastname', 'real_age', 'dob',
"Date Parents Contacted", "Parents Preference",
"Date Leaders Contacted", "Agreed Section",
"Starting Date", "Leaving Date", "Notes", "Priority",
'8', '10 1/2', '14 1/2']
movers = section_.movers
if age:
threshold = (365 * float(age))
now = datetime.datetime.now()
age_fn = lambda dob: (now - datetime.datetime.strptime(dob, '%Y-%m-%d')).days
movers = [mover for mover in section_.movers
if age_fn(mover['dob']) > threshold]
now = datetime.datetime.now()
for mover in movers:
real_dob = datetime.datetime.strptime(mover['dob'], '%Y-%m-%d')
rel_age = relativedelta.relativedelta(now, real_dob)
mover['real_age'] = "{0:02d}.{0:02d}".format(rel_age.years, rel_age.months)
mover['8'] = (real_dob+relativedelta.relativedelta(years=8)).strftime("%b %y")
mover['10 1/2'] = (real_dob + relativedelta.relativedelta(years=10, months=6)).strftime("%b %y")
mover['14 1/2'] = (real_dob + relativedelta.relativedelta(years=14, months=6)).strftime("%b %y")
rows += [[section_['sectionname']] +
[member[header] for header in headers]
for member in movers]
headers = ["Current Section"] + headers
if csv:
w = csv_writer(sys.stdout)
if not no_headers:
w.writerow(list(headers))
w.writerows(rows)
else:
if not no_headers:
print(tabulate.tabulate(rows, headers=headers))
else:
print(tabulate.tabulate(rows, tablefmt="plain"))
def _main(osm, auth, sections, outdir, term):
assert os.path.exists(outdir) and os.path.isdir(outdir)
group = Group(osm, auth, MAPPING.keys(), term)
for section in sections:
assert section in group.SECTIONIDS.keys(), \
"section must be in {!r}.".format(group.SECTIONIDS.keys())
for section in sections:
vcards = [member2vcard(member, section) for
member in group.section_all_members(section)]
open(os.path.join(outdir, section + ".vcf"), 'w').writelines(vcards)
def _main(osm, auth, sections, outdir, term):
assert os.path.exists(outdir) and os.path.isdir(outdir)
group = Group(osm, auth, MAPPING.keys(), term)
for section in sections:
assert section in group.SECTIONIDS.keys(), \
"section must be in {!r}.".format(group.SECTIONIDS.keys())
for section in sections:
entries = [member2compass(member, section) for
member in group.section_yp_members_without_leaders(section)]
[check(entry, section) for entry in entries]
with open(os.path.join(outdir, section + ".csv"), "w") as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=compass_headings)
writer.writeheader()
[writer.writerow(entry) for entry in entries]
def events_attendees(osm, auth, sections, event,
term=None, csv=False, attending_only=False,
no_headers=False):
group = Group(osm, auth, MAPPING.keys(), term)
for section in sections:
section_ = group._sections.sections[Group.SECTIONIDS[section]]
ev = section_.events.get_by_name(event)
if not ev:
log.error("No such event: {}".format(event))
sys.exit(0)
attendees = ev.attendees
mapping = ev.fieldmap
if attending_only:
attendees = [attendee for attendee in attendees
if attendee['attending'] == "Yes"]
extra_fields = {
'patrol': 'Six',
'age': 'Age',
}
def fields(attendee):
out = [str(attendee[_[1]]) for _ in mapping] + \
[section_.members.get_by_event_attendee(attendee)[_] for _ in
extra_fields.keys()]
return out
output = [fields(attendee)
for attendee in attendees if section_.members.is_member(attendee['scoutid'])]
headers = [_[0] for _ in mapping] + list(extra_fields.values())
if csv:
w = csv_writer(sys.stdout)
if not no_headers:
w.writerow(list(headers))
w.writerows(output)
else:
if not no_headers:
print(tabulate.tabulate(output, headers=headers))
else:
print(tabulate.tabulate(output, tablefmt="plain"))
def contacts_detail(osm, auth, sections, csv=False, term=None, no_headers=False):
group = Group(osm, auth, MAPPING.keys(), term)
section_map = {'Garrick': 'Beavers',
'Paget': 'Beavers',
'Swinfen': 'Beavers',
'Maclean': 'Cubs',
'Somers': 'Cubs',
'Rowallan': 'Cubs',
'Erasmus': 'Scouts',
'Boswell': 'Scouts',
'Johnson': 'Scouts'}
rows = []
def add_row(section, member):
rows.append([section_map[section], section, member['first_name'], member['last_name'],
member['date_of_birth'],
member['contact_primary_1.email1'],
member['contact_primary_1.address1'],
member['contact_primary_1.address2'],
member['contact_primary_1.address3'],
member['contact_primary_1.postcode'],
member['contact_primary_2.address1'],
member['floating.gender'].lower()])
for section in sections:
for member in group.section_all_members(section):
add_row(section, member)
headers = ["Section", "Section Name", "First", "Last", "DOB", "Email1", "Address1", "Address1.1", "Address1.2", "Address1.3",
"Address2", "Address3", "Gender"]
if csv:
w = csv_writer(sys.stdout)
if not no_headers:
w.writerow(list(headers))
w.writerows(rows)
else:
if not no_headers:
print(tabulate.tabulate(rows, headers=headers))
else:
print(tabulate.tabulate(rows, tablefmt="plain"))
def _main(osm, auth, sections, outdir, email, term):
assert os.path.exists(outdir) and os.path.isdir(outdir)
group = Group(osm, auth, MAPPING.keys(), term)
for section in sections:
assert section in group.SECTIONIDS.keys(), \
"section must be in {!r}.".format(group.SECTIONIDS.keys())
vcards = []
for section in sections:
section_vcards = [member2vcards(member, section) for
member in group.section_all_members(section)]
# flatten list of lists.
vcards += list(itertools.chain(*section_vcards))
open(os.path.join(outdir, "group.vcf"), 'w').writelines(vcards)
if email:
send([email, ], "OSM Group vcards", "".join(vcards))
def members_badges(osm, auth, sections, csv=False, no_headers=False, term=None):
group = Group(osm, auth, MAPPING.keys(), term)
for section in sections:
# members = group._sections.sections[Group.SECTIONIDS[section]].members
members = group.section_yp_members_without_leaders(section)
rows = []
for member in members:
badges = member.get_badges(section_type=group.SECTION_TYPE[section])
if badges:
# If no badges - probably a leader
challenge_new = len([badge for badge in badges
if badge['awarded'] == '1' and badge['badge_group'] == '1'
and not badge['badge'].endswith('(Pre 2015)')])
challenge_old = len([badge for badge in badges
if badge['awarded'] == '1' and badge['badge_group'] == '1'
and badge['badge'].endswith('(Pre 2015)')])
activity = len([badge for badge in badges if badge['awarded'] == '1' and badge['badge_group'] == '2'])
staged = len([badge for badge in badges if badge['awarded'] == '1' and badge['badge_group'] == '3'])
core = len([badge for badge in badges if badge['awarded'] == '1' and badge['badge_group'] == '4'])
rows.append([member['date_of_birth'], member['last_name'], member['age'], section,
challenge_new, challenge_old, activity, staged, core])
headers = ["DOB", "Last Name", "Age", "Section Name", "Challenge", "Challenge_old", "Staged", "Activity", "Core"]
if csv:
w = csv_writer(sys.stdout)
if not no_headers:
w.writerow(list(headers))
w.writerows(rows)
else:
if not no_headers:
print(tabulate.tabulate(rows, headers=headers))
else:
print(tabulate.tabulate(rows, tablefmt="plain"))
def sync_contacts(osm, auth, sections, google_accounts,
csv=False, term=None, no_headers=False,
delete_google_groups=False):
group = Group(osm, auth, MAPPING.keys(), term)
section_map = Group.SECTION_TYPE
contacts = {}
def get(field, member, section):
return member["{}.{}".format(section, field)]
def get_address(func):
return ",".join([func('address1'),
func('address2'),
func('address3'),
func('postcode')])
def add_member_contacts(section, section_type, member):
f1 = functools.partial(get, member=member, section='contact_primary_member')
custom_field = functools.partial(get, member=member, section='customisable_data')
first = string.capwords(member['first_name'].strip())
last_ = string.capwords(member['last_name'].strip())
last = f'{last_} (OSM)'
full_name_osm = f'{first} {last}'
full_name = f'{first} {last}'
section_capped = string.capwords(section)
yp_name = string.capwords(f"{first} {last_} ({section_capped} {section_type})")
parent_groups = [string.capwords(_) for _ in [f'{section} Parents', f'{section_type} Parents',
f'All Parents', 'All']]
yl_groups = [string.capwords(_) for _ in
[f'{section} Young Leaders', f'{section_type} Young Leaders',
f'All Young Leaders', 'All Adults and Young Leaders', 'All']]
leader_groups = [string.capwords(_) for _ in
[f'{section} Leaders', f'{section_type} Leaders',
'All Leaders', 'All Adults and Young Leaders',
'All']]
all_adults_and_young_leaders_groups = ['All Adults And Young Leaders', 'All']
exec_groups = ['Exec Members', 'All']
def is_valid_email(func, tag):
email_ = func(tag).strip()
return (len(email_) != 0
and (not email_.startswith('x ') and
func("{}_leaders".format(tag)) == "yes"))
def parse_tel(number_field, default_name):
index = 0
for i in range(len(number_field)):
if number_field[i] not in ['0', '1', '2', '3', '4', '5',
'6', '7', '8', '9', ' ', '\t']:
index = i
break
number_ = number_field[:index].strip() if index != 0 else number_field
name_ = number_field[index:].strip() if index != 0 else default_name
# print("input = {}, index = {}, number = {}, name = {}".format(
# number_field, index, number, name))
return number_, name_
# Add member data
email = f1('email1').strip()
if group.is_leader(member) and not is_valid_email(f1, 'email1'):
log.warning(f'{full_name} is a leader but does not have a member email address.')
if is_valid_email(f1, 'email1'):
key = '{}-{}-{}'.format(first.lower(), last.lower(), email.lower())
address = get_address(f1)
if key not in contacts:
contacts[key] = dict(first=first,
last=last,
email=email.lower(),
addresses=[],
yp=[],
sections=[],
groups=[],
phones=[])
contacts[key]['sections'].append(section)
if not group.is_leader(member):
contacts[key]['yp'].append(yp_name)
if section.lower() == 'adult':
# Everyone in the adult group is either a leader, a non-leader adult or a YL.
contacts[key]['groups'].extend(all_adults_and_young_leaders_groups)
if custom_field('cf_exec').lower() in ['y', 'yes']:
contacts[key]['groups'].extend(exec_groups)
else:
# If we are not in the adult group, we must be in a normal section group
# so, if they are an adult they must be a leader.
if group.is_leader(member):
contacts[key]['groups'].extend(leader_groups)
if group.is_yl(member):
contacts[key]['groups'].extend(yl_groups)
if len(address):
contacts[key]['addresses'].append(('member', address))
for _ in ['phone1', 'phone2']:
number, name = parse_tel(f1(_), _)
if number != "":
contacts[key]['phones'].append((number, name))
# For all we add the contact details if they say they want to be contacted
if not (section.lower() == 'adult' or group.is_leader(member)):
for _ in ('contact_primary_1', 'contact_primary_2'):
f = functools.partial(get, member=member, section=_)
_email = f('email1').strip()
if is_valid_email(f, 'email1'):
_first = f('firstname').strip()
_last = f('lastname').strip() if len(f('lastname').strip()) > 0 else last_
_last_osm = f'{_last} (OSM)'
key = '{}-{}-{}'.format(_first.lower(), _last_osm.lower(), _email.lower())
address = get_address(f)
if key not in contacts:
contacts[key] = dict(first=_first,
last=_last_osm,
email=_email.lower(),
addresses=[],
yp=[],
sections=[],
groups=[],
phones=[])
contacts[key]['sections'].append(section)
contacts[key]['yp'].append(yp_name)
contacts[key]['groups'].extend(parent_groups)
if len(address):
contacts[key]['addresses'].append((f'{_first} {_last}', address))
if group.is_yl(member):
contacts[key]['groups'].extend(yl_groups)
for _ in ['phone1', 'phone2']:
number, name = parse_tel(f(_), _)
if number != "":
contacts[key]['phones'].append((number, name))
# For every member in the group we want to extract each unique contact address. We ignore all those that
# not marked for wanting contact.
# We then add all of the these unique contacts to google.
# Now we add each of the contacts to the groups that they are associated with.
log.info("Fetch members from OSM")
for section_ in sections:
for member in group.section_all_members(section_):
add_member_contacts(section_, Group.SECTION_TYPE[section_], member)
# for c in contacts.values():
# print(f'{c}')
# remove duplicates
groups = []
for key, contact in contacts.items():
contact['groups'] = set(contact['groups'])
contact['sections'] = set(contact['sections'])
contact['yp'] = set(contact['yp'])
contact['addresses'] = set(contact['addresses'])
contact['phones'] = set(contact['phones'])
groups.extend(contact['groups'])
# Gather all the groups
groups = set(groups)
group_names = [f"{_} (OSM)" for _ in groups]
contacts = [contact for key, contact in contacts.items()]
# Sync up the google groups.
log.info("Fetch list of groups from google")
existing_osm_groups = fetch_list_of_groups()
# Fetch the list of group-members - this is used to find who should
# be managers of the groups.
existing_role_group_members = fetch_group_members(
fetch_list_of_groups(prefix='7th-'))
if delete_google_groups:
for group in existing_osm_groups:
log.info(f"Deleting group: {group}")
delete_google_group(group)
existing_osm_groups = fetch_list_of_groups() # should return empty
missing_groups = [_ for _ in groups if
convert_group_name_to_email_address(_) not in existing_osm_groups]
if len(missing_groups) > 0:
create_groups(missing_groups)
for group in groups:
group_email_address = convert_group_name_to_email_address(group)
group_moderators = get_group_moderaters(group_email_address,
existing_role_group_members)
group_members = [contact for contact in contacts
if (group in contact['groups'] and
contact['email'] not in group_moderators)]
if len(group_moderators) == 0:
log.warning(f'No managers for group: {group_email_address}')
if len(group_members) == 0:
log.warning(f'No members in group: {group}')
log.info(f"Syncing contacts in google group: {group}")
sync_contacts_in_group(
group_email_address,
[_['email'] for _ in group_members])
sync_contacts_in_group(
group_email_address, group_moderators, role='manager')
# Sync all contacts to the global directory.
log.info("delete OSM contacts in directory")
delete_osm_contacts_already_in_gam()
log.info("Create all contacts in directory")
for contact in contacts:
create_osm_contact_in_gam(contact)
for google_account in google_accounts:
if True: # Flip this to false to manually remove all contacts from the users.
log.info(f"Syncing OSM contacts into google account for: {google_account}")
# Setup and sync all contacts and contact groups to a user.
existing_groups = fetch_list_of_contact_groups(google_account)
create_contact_groups([_ for _ in group_names if _ not in existing_groups], google_account)
delete_osm_contacts_already_in_google(google_account)
for contact in contacts:
create_osm_contact_in_google(contact, google_account)
else:
# To remove all contacts and contact groups from a user.
existing_groups = fetch_list_of_contact_groups(google_account, field='ContactGroupID')
delete_osm_contacts_already_in_google(google_account)
delete_contact_groups(existing_groups, google_account)
log.info("Finished.")
def census_list(osm, auth, term=None, csv=False, attending_only=False,
no_headers=False):
group = Group(osm, auth, MAPPING.keys(), term)
section_map = {'Garrick': 'Beavers',
'Paget': 'Beavers',
'Swinfen': 'Beavers',
'Maclean': 'Cubs',
'Somers': 'Cubs',
'Rowallan': 'Cubs',
'Erasmus': 'Scouts',
'Boswell': 'Scouts',
'Johnson': 'Scouts'}
rows = []
def add_row(section, member):
rows.append([section_map[section], section, member['first_name'], member['last_name'],
member['date_of_birth'],
member['contact_primary_member.address1'],
member['contact_primary_1.address1'],
member['contact_primary_1.address2'],
member['contact_primary_1.address3'],
member['contact_primary_1.postcode'],
member['contact_primary_2.address1'],
member['floating.gender'].lower()])
all_members_dict = group.all_yp_members_without_senior_duplicates_dict()
for section in ('Swinfen', 'Paget', 'Garrick'):
members = all_members_dict[section]
for member in members:
age = member.age().days / 365
if (age > 5 and age < 9):
add_row(section, member)
else:
log.info("Excluding: {} {} because not of Beaver age ({}).".format(
member['first_name'], member['last_name'], age
))
for section in ('Maclean', 'Rowallan', 'Somers'):
members = all_members_dict[section]
for member in members:
age = member.age().days / 365
if (age > 7 and age < 11):
add_row(section, member)
else:
log.info("Excluding: {} {} because not of Cub age ({}).".format(
member['first_name'], member['last_name'], age
))
for section in ('Johnson', 'Boswell', 'Erasmus'):
members = all_members_dict[section]
for member in members:
age = member.age().days / 365
if (age > 10 and age < 16):
add_row(section, member)
else:
log.info("Excluding: {} {} because not of Scout age ({}).".format(
member['first_name'], member['last_name'], age
))
headers = ["Section", "Section Name", "First", "Last", "DOB", "Address1", "Address1.1", "Address1.2", "Address1.3",
"Address2", "Address3", "Gender"]
if csv:
w = csv_writer(sys.stdout)
if not no_headers:
w.writerow(list(headers))
w.writerows(rows)
else:
if not no_headers:
print(tabulate.tabulate(rows, headers=headers))
else:
print(tabulate.tabulate(rows, tablefmt="plain"))
def census_leavers(osm, auth, term=None, csv=False,
no_headers=False):
# Nasty hack - but I need a list of terms.
somers_terms = Group(osm, auth, MAPPING.keys(), None)._sections.sections['20706'].get_terms()
def find_term(name):
return [_ for _ in somers_terms if _['name'] == name][0]
terms = [find_term(_) for _ in
['Summer 2014',
'Autumn 2014',
'Spring 2015',
'Summer 2015',
'Autumn 2015',
'Spring 2016',
'Summer 2016',
'Autumn 2016',
'Spring 2017',
'Summer 2017',
'Autumn 2017',
]]
pairs = [(terms[x], terms[x + 1]) for x in range(len(terms) - 1)]
section_map = {'Garrick': 'Beavers',
'Paget': 'Beavers',
'Swinfen': 'Beavers',
'Maclean': 'Cubs',
'Somers': 'Cubs',
'Rowallan': 'Cubs',
'Erasmus': 'Scouts',
'Boswell': 'Scouts',
'Johnson': 'Scouts'}
rows = []
for old, new in pairs:
old_term = Group(osm, auth, MAPPING.keys(), old['name'])
new_term = Group(osm, auth, MAPPING.keys(), new['name'])
old_members_raw = old_term.all_yp_members_without_senior_duplicates()
new_members_raw = new_term.all_yp_members_without_senior_duplicates()
old_members = [(_['first_name'], _['last_name'])
for _ in old_members_raw]
new_members = [(_['first_name'], _['last_name'])
for _ in new_members_raw]
missing = [_ for _ in old_members if not new_members.count(_)]
for first, last in missing:
sections = old_term.find_sections_by_name(first, last)
member = old_members_raw[old_members.index((first, last))]
age = member.age(ref_date=old.enddate).days // 365
rows.append([old['name'], section_map[sections[0]], sections[0], first, last, age, member['date_of_birth'],
member['floating.gender'].lower()])
headers = ["Last Term", "Section", "Section Name", "First", "Last", "Age", "DOB", "Gender"]
if csv:
w = csv_writer(sys.stdout)
if not no_headers:
w.writerow(list(headers))
w.writerows(rows)
else:
if not no_headers:
print(tabulate.tabulate(rows, headers=headers))
else:
print(tabulate.tabulate(rows, tablefmt="plain"))
def contacts_list(osm, auth, sections, term=None):
group = Group(osm, auth, MAPPING.keys(), term)
for section in sections:
for member in group.section_all_members(section):
print("{} {}".format(member['first_name'], member['last_name']))
def users_list(osm, auth, sections, csv=False, no_headers=False, term=None):
group = Group(osm, auth, MAPPING.keys(), term)
for section in sections:
for user in group._sections.sections[Group.SECTIONIDS[section]].users:
print(user['firstname'])
def events_list(osm, auth, sections, term=None):
group = Group(osm, auth, MAPPING.keys(), term)
for section in sections:
for event in group._sections.sections[Group.SECTIONIDS[section]].events:
print(event['name'])
