def on_markdown_context(context, page, config):
context['page'] = page
context['config'] = config
context['club_elapsed_months'] = int(
round((context['now'] - CLUB_LAUNCH_AT).days / 30))
context['members'] = ClubUser.avatars_listing()
context['members_total_count'] = ClubUser.members_count()
if 'topic_name' in page.meta:
topic_name = page.meta['topic_name']
context['topic'] = Topic.get_by_id(topic_name)
def define_env(env):
with db:
env.variables.update(dict(
members=ClubUser.avatars_listing(),
members_total_count=ClubUser.members_count(),
jobs_count=Job.aggregate_metrics()['jobs_count'],
))
@env.filter
def sample(items, n=2, sample_fn=None): # TODO deduplicate template_filters
items = list(items)
if len(items) <= n:
return items
return (sample_fn or random.sample)(items, n)
async def apply_changes(client, changes):
all_discord_roles = {
discord_role.id: discord_role
for discord_role in await client.juniorguru_guild.fetch_roles()
}
changes_by_members = {}
for member_id, op, role_id in changes:
changes_by_members.setdefault(member_id, dict(add=[], remove=[]))
changes_by_members[member_id][op].append(role_id)
for member_id, changes in changes_by_members.items():
discord_member = await client.juniorguru_guild.fetch_member(member_id)
if changes['add']:
discord_roles = [
all_discord_roles[role_id] for role_id in changes['add']
]
log.info(
f'{discord_member.display_name}: adding {repr_roles(discord_roles)}'
)
await discord_member.add_roles(*discord_roles)
if changes['remove']:
discord_roles = [
all_discord_roles[role_id] for role_id in changes['remove']
]
log.info(
f'{discord_member.display_name}: removing {repr_roles(discord_roles)}'
)
await discord_member.remove_roles(*discord_roles)
with db:
member = ClubUser.get_by_id(member_id)
member.roles = get_roles(discord_member)
member.save()
def test_avatars_listing(db_connection):
user1 = create_user(1, avatar_path='avatars/1.png')
user2 = create_user(2) # noqa
user3 = create_user(3, avatar_path='avatars/2.png')
user4 = create_user(4, avatar_path='avatars/3.png')
assert list(ClubUser.avatars_listing()) == [user1, user3, user4]
def test_author_members_listing(db_connection):
author1 = create_user(1, is_member=True, is_bot=True) # noqa
author2 = create_user(2, is_member=True, is_bot=False)
author3 = create_user(3, is_member=False, is_bot=True) # noqa
author4 = create_user(4, is_member=False, is_bot=False) # noqa
assert list(ClubUser.members_listing()) == [author2]
def create_user(id_, **kwargs):
return ClubUser.create(id=id_,
is_member=kwargs.get('is_member', True),
is_bot=kwargs.get('is_bot', False),
avatar_path=kwargs.get('avatar_path'),
display_name=kwargs.get('display_name',
'Kuře Žluté'),
mention=kwargs.get('mention', f'<@{id_}>'),
joined_at=kwargs.get(
'joined_at',
datetime.now() - timedelta(days=3)))
async def main(client):
AVATARS_PATH.mkdir(exist_ok=True, parents=True)
for path in AVATARS_PATH.glob('*.png'):
path.unlink()
with db:
for member in ClubUser.members_listing():
log.info(
f"Downloading avatar for '{member.display_name}' #{member.id}")
discord_member = await client.juniorguru_guild.fetch_member(
member.id)
member.avatar_path = await download_avatar(discord_member)
log.info(f"Result: '{member.avatar_path}'")
member.save()
def main():
with db:
members = ClubUser.members_listing()
changes = []
top_members_limit = ClubUser.top_members_limit()
log.info(
f'members_count={len(members)}, top_members_limit={top_members_limit}'
)
# ROLE_MOST_DISCUSSING
messages_count_stats = calc_stats(members,
lambda m: m.messages_count(),
top_members_limit)
log.info(f"messages_count {repr_stats(members, messages_count_stats)}")
recent_messages_count_stats = calc_stats(
members, lambda m: m.recent_messages_count(), top_members_limit)
log.info(
f"recent_messages_count {repr_stats(members, recent_messages_count_stats)}"
)
most_discussing_members_ids = set(messages_count_stats.keys()) | set(
recent_messages_count_stats.keys())
log.info(
f"most_discussing_members: {repr_ids(members, most_discussing_members_ids)}"
)
for member in members:
changes.extend(
evaluate_changes(member.id, member.roles,
most_discussing_members_ids,
ROLE_MOST_DISCUSSING))
# ROLE_MOST_HELPFUL
upvotes_count_stats = calc_stats(members, lambda m: m.upvotes_count(),
top_members_limit)
log.info(f"upvotes_count {repr_stats(members, upvotes_count_stats)}")
recent_upvotes_count_stats = calc_stats(
members, lambda m: m.recent_upvotes_count(), top_members_limit)
log.info(
f"recent_upvotes_count {repr_stats(members, recent_upvotes_count_stats)}"
)
most_helpful_members_ids = set(upvotes_count_stats.keys()) | set(
recent_upvotes_count_stats.keys())
log.info(
f"most_helpful_members: {repr_ids(members, most_helpful_members_ids)}"
)
for member in members:
changes.extend(
evaluate_changes(member.id, member.roles,
most_helpful_members_ids, ROLE_MOST_HELPFUL))
# ROLE_HAS_INTRO_AND_AVATAR
intro_avatar_members_ids = [
member.id for member in members
if member.avatar_path and member.has_intro()
]
log.info(
f"intro_avatar_members: {repr_ids(members, intro_avatar_members_ids)}"
)
for member in members:
changes.extend(
evaluate_changes(member.id, member.roles,
intro_avatar_members_ids,
ROLE_HAS_INTRO_AND_AVATAR))
# ROLE_IS_NEW
new_members_ids = [member.id for member in members if member.is_new()]
log.info(f"new_members_ids: {repr_ids(members, new_members_ids)}")
for member in members:
changes.extend(
evaluate_changes(member.id, member.roles, new_members_ids,
ROLE_IS_NEW))
# ROLE_IS_SPEAKER
speaking_members_ids = [
member.id for member in Event.list_speaking_members()
]
log.info(
f"speaking_members_ids: {repr_ids(members, speaking_members_ids)}")
for member in members:
changes.extend(
evaluate_changes(member.id, member.roles, speaking_members_ids,
ROLE_IS_SPEAKER))
if DISCORD_MUTATIONS_ENABLED:
apply_changes(changes)
else:
log.warning(
"Skipping Discord mutations, DISCORD_MUTATIONS_ENABLED not set")
def list_speaking_members(cls):
return ClubUser.select() \
.where(ClubUser.is_member == True) \
.join(EventSpeaking)
async def main(client):
with db:
top_members_limit = ClubUser.top_members_limit()
pin_reactions_by_members = [
pin_reaction for pin_reaction in ClubPinReaction.listing()
if pin_reaction.user.is_member
]
log.info(
f'Found {len(pin_reactions_by_members)} pin reactions by people who are currently members'
)
for pin_reaction in pin_reactions_by_members:
member = await client.juniorguru_guild.fetch_member(
pin_reaction.user.id)
if member.dm_channel:
channel = member.dm_channel
else:
channel = await member.create_dm()
log.info(
f"Checking DM if already pinned for {member.display_name} #{member.id}"
)
if await is_pinned(pin_reaction.message.url, channel):
log.info(f"Already pinned for {member.display_name} #{member.id}")
continue
log.info(
f"Not pinned for {member.display_name} #{member.id}, sending a message to DM"
)
if DISCORD_MUTATIONS_ENABLED:
content = (
'📌 VidÃm Å¡pendlÃk! Ukládám ti pÅ™ÃspÄ›vek sem, do soukromé zprávy. '
f'Když bude mÃt ~{top_members_limit} Å¡pendlÃků, pÅ™ipnu jej v původnÃm kanálu pro vÅ¡echny.'
)
embed_description = [
f"{pin_reaction.message.author.mention} v {pin_reaction.message.channel_mention}:",
f"> {textwrap.shorten(pin_reaction.message.content, 500, placeholder='…')}",
f"[Hop na pÅ™ÃspÄ›vek]({pin_reaction.message.url})",
"",
]
await channel.send(
content=content,
embed=Embed(description="\n".join(embed_description)))
else:
log.warning(
"Skipping Discord mutations, DISCORD_MUTATIONS_ENABLED not set"
)
log.info(
f"Going through messages pinned by reactions, minimum is {top_members_limit} pin reactions"
)
with db:
messages = ClubMessage.pinned_by_reactions_listing(top_members_limit)
log.info(f'Found {len(messages)} messages')
for message in messages:
log.info(
f"Message {message.url} {'PINNED' if message.is_pinned else 'NOT PINNED'}"
)
if not message.is_pinned:
log.info(f"Pinning {message.url}")
discord_message = await client.fetch_channel(
message.channel_id).fetch_message(message.id)
await discord_message.pin(
reason=
f"The message has {message.pin_reactions_count} pin reactions, minimum is {top_members_limit}"
)
async def main(client):
with db:
db.drop_tables([ClubMessage, ClubUser, ClubPinReaction])
db.create_tables([ClubMessage, ClubUser, ClubPinReaction])
authors = {}
relevant_channels = (channel for channel in client.juniorguru_guild.text_channels
if channel.permissions_for(client.juniorguru_guild.me).read_messages)
for channel in relevant_channels:
log.info(f'Channel #{channel.name}')
async for message in channel.history(limit=None, after=None):
if message.author.id not in authors:
# The message.author can be an instance of Member, but it can also be an instance of User,
# if the author isn't a member of the Discord guild/server anymore. User instances don't
# have certain properties, hence the getattr() calls below.
with db:
log.info(f"User '{message.author.display_name}' #{message.author.id}")
author = ClubUser.create(id=message.author.id,
is_bot=message.author.bot,
is_member=bool(getattr(message.author, 'joined_at', False)),
display_name=message.author.display_name,
mention=message.author.mention,
joined_at=getattr(message.author, 'joined_at', None),
roles=get_roles(message.author))
authors[message.author.id] = author
with db:
ClubMessage.create(id=message.id,
url=message.jump_url,
content=message.content,
upvotes_count=count_upvotes(message.reactions),
pin_reactions_count=count_pins(message.reactions),
created_at=message.created_at,
edited_at=message.edited_at,
author=authors[message.author.id],
channel_id=channel.id,
channel_name=channel.name,
channel_mention=channel.mention,
type=message.type.name)
users = set()
for reaction in message.reactions:
if emoji_name(reaction.emoji) in EMOJI_PINS:
for user in [user async for user in reaction.users()]:
users.add(user)
for user in users:
log.info(f"Message {message.jump_url} is pinned by user '{user.display_name}' #{user.id}")
ClubPinReaction.create(user=user.id, message=message.id)
# remaining members (did not author a single message)
log.info('Looking for remaining members, if any')
remaining_members = [member async for member in client.juniorguru_guild.fetch_members(limit=None)
if member.id not in authors]
for member in remaining_members:
with db:
log.info(f"Member '{member.display_name}' #{member.id}")
ClubUser.create(id=member.id,
is_bot=member.bot,
is_member=True,
display_name=member.display_name,
mention=member.mention,
joined_at=member.joined_at,
roles=get_roles(member))
with db:
messages_count = ClubMessage.count()
members_count = ClubUser.members_count()
log.info(f'Saved {messages_count} messages from {len(authors)} authors')
log.info(f'Saved {members_count} members')
def test_author_top_members_limit_rounds_up(db_connection):
create_user(1)
create_user(2)
create_user(3)
ClubUser.top_members_limit() == 1
def test_author_top_members_limit_doesnt_count_bots(db_connection):
for id_ in range(200):
create_user(id_, is_bot=id_ < 100)
ClubUser.top_members_limit() == 5
def test_author_top_members_limit_is_five_percent(db_connection):
for id_ in range(100):
create_user(id_)
ClubUser.top_members_limit() == 5
def main():
log.info('Getting data from Memberful')
# https://memberful.com/help/integrate/advanced/memberful-api/
# https://juniorguru.memberful.com/api/graphql/explorer?api_user_id=52463
transport = RequestsHTTPTransport(
url='https://juniorguru.memberful.com/api/graphql/',
headers={'Authorization': f'Bearer {MEMBERFUL_API_KEY}'},
verify=True,
retries=3)
memberful = Memberful(transport=transport,
fetch_schema_from_transport=True)
query = gql("""
query getSubscriptions($cursor: String!) {
subscriptions(after: $cursor) {
totalCount
pageInfo {
endCursor
hasNextPage
}
edges {
node {
active
createdAt
expiresAt
pastDue
coupon {
code
}
orders {
createdAt
coupon {
code
}
}
member {
discordUserId
email
fullName
id
stripeCustomerId
}
}
}
}
}
""")
records = []
seen_discord_ids = set()
cursor = ''
while cursor is not None:
log.info('Requesting Memberful GraphQL')
params = dict(cursor=cursor)
result = memberful.execute(query, params)
for edge in result['subscriptions']['edges']:
node = edge['node']
discord_id = node['member']['discordUserId']
user = None
if discord_id:
seen_discord_ids.add(discord_id)
try:
user = ClubUser.get_by_id(int(discord_id))
except ClubUser.DoesNotExist:
pass
records.append({
'Name':
node['member']['fullName'],
'Discord Name':
user.display_name if user else None,
'E-mail':
node['member']['email'],
'Memberful ID':
node['member']['id'],
'Stripe ID':
node['member']['stripeCustomerId'],
'Discord ID':
discord_id,
'Memberful Active?':
node['active'],
'Memberful Since':
arrow.get(node['createdAt']).date().isoformat(),
'Memberful End':
arrow.get(node['expiresAt']).date().isoformat(),
'Memberful Coupon':
get_active_coupon(node),
'Discord Member?':
user.is_member if user else False,
'Discord Since':
user.first_seen_on().isoformat() if user else None,
'Memberful Past Due?':
node['pastDue'],
})
if result['subscriptions']['pageInfo']['hasNextPage']:
cursor = result['subscriptions']['pageInfo']['endCursor']
else:
cursor = None
log.info('Process remaining Discord users')
for user in ClubUser.listing():
discord_id = str(user.id)
if not user.is_bot and discord_id not in seen_discord_ids:
records.append({
'Name': None,
'Discord Name': user.display_name,
'E-mail': None,
'Memberful ID': None,
'Stripe ID': None,
'Discord ID': discord_id,
'Memberful Active?': False,
'Memberful Since': None,
'Memberful End': None,
'Memberful Coupon': None,
'Discord Member?': user.is_member,
'Discord Since': user.first_seen_on().isoformat(),
'Memberful Past Due?': False,
})
log.info('Uploading subscriptions to Google Sheets')
google_sheets.upload(google_sheets.get(DOC_KEY, 'subscriptions'), records)