def main():
# If the creation of the directory is left to the spiders, they can end
# up colliding in making sure it gets created
Path(IMAGES_STORE).mkdir(exist_ok=True, parents=True)
with db:
db.drop_tables([Job, JobError, JobDropped, SpiderMetric])
db.create_tables([Job, JobError, JobDropped, SpiderMetric])
spider_names = [
'juniorguru',
'linkedin',
'stackoverflow',
'startupjobs',
'remoteok',
'wwr',
'dobrysef',
]
Pool().map(run_spider, spider_names)
manage_jobs_channel()
def main():
path = DATA_DIR / 'events.yml'
records = [
load_record(record.data) for record in load(path.read_text(), schema)
]
if FLUSH_POSTERS:
log.warning("Removing all existing posters, FLUSH_POSTERS is set")
for poster_path in POSTERS_DIR.glob('*.png'):
poster_path.unlink()
with db:
db.drop_tables([Event, EventSpeaking])
db.create_tables([Event, EventSpeaking])
# process data from the YAML, generate posters
for record in records:
name = record['title']
log.info(f"Creating '{name}'")
speakers_ids = record.pop('speakers', [])
event = Event.create(**record)
for speaker_id in speakers_ids:
try:
avatar_path = next(
(IMAGES_DIR / 'speakers').glob(f"{speaker_id}.*"))
except StopIteration:
log.info(f"Didn't find speaker avatar for {speaker_id}")
avatar_path = None
else:
log.info(f"Downsizing speaker avatar for {speaker_id}")
avatar_path = downsize_square_photo(
avatar_path, 500).relative_to(IMAGES_DIR)
log.info(f"Marking member {speaker_id} as a speaker")
EventSpeaking.create(speaker=speaker_id,
event=event,
avatar_path=avatar_path)
if event.logo_path:
log.info(f"Checking '{event.logo_path}'")
image_path = IMAGES_DIR / event.logo_path
if not image_path.exists():
raise ValueError(
f"Event '{name}' references '{image_path}', but it doesn't exist"
)
log.info(f"Rendering poster for '{name}'")
context = dict(
event=model_to_dict(event, extra_attrs=['first_avatar_path']))
image_path = render_image_file('poster.html',
context,
POSTERS_DIR,
filters={
'md': md,
'local_time': local_time,
'weekday': weekday,
})
event.poster_path = image_path.relative_to(IMAGES_DIR)
event.save()
log.info(f"Rendering Instagram poster for '{name}'")
save_as_ig_square(image_path)
# discord messages
if DISCORD_MUTATIONS_ENABLED:
post_next_event_messages()
else:
log.warning(
"Skipping Discord mutations, DISCORD_MUTATIONS_ENABLED not set"
)
async def main(client):
# MESSAGES AND AUTHORS
with db:
db.drop_tables([Message, MessageAuthor])
db.create_tables([Message, MessageAuthor])
authors = {}
relevant_channels = (channel for channel in client.juniorguru_guild.text_channels
if channel.permissions_for(client.juniorguru_guild.me).read_messages)
for channel in relevant_channels:
log.info(f'#{channel.name}')
async for message in channel.history(limit=None, after=None):
if message.author.id not in authors:
# The message.author can be an instance of Member, but it can also be an instance of User,
# if the author isn't a member of the Discord guild/server anymore. User instances don't
# have certain properties, hence the getattr() calls below.
with db:
author = MessageAuthor.create(id=message.author.id,
is_bot=message.author.bot,
is_member=bool(getattr(message.author, 'joined_at', False)),
has_avatar=not is_default_avatar(message.author.avatar_url),
display_name=message.author.display_name,
mention=message.author.mention,
joined_at=getattr(message.author, 'joined_at', None),
roles=get_roles(message.author))
authors[message.author.id] = author
with db:
Message.create(id=message.id,
url=message.jump_url,
content=message.content,
upvotes=count_upvotes(message.reactions),
created_at=message.created_at,
edited_at=message.edited_at,
author=authors[message.author.id],
channel_id=channel.id,
channel_name=channel.name,
channel_mention=channel.mention,
type=message.type.name)
with db:
messages_count = Message.count()
log.info(f'Saved {messages_count} messages from {len(authors)} authors')
# RETURNING MEMBERS
system_messages_channel = await client.fetch_channel(SYSTEM_MESSAGES_CHANNEL)
for message in Message.channel_listing(SYSTEM_MESSAGES_CHANNEL):
if message.type == 'new_member' and message.author.first_seen_at() < message.created_at.date():
log.info(f'It looks like {message.author.display_name} has returned')
discord_message = await system_messages_channel.fetch_message(message.id)
if DISCORD_MUTATIONS_ENABLED:
await discord_message.add_reaction('👋')
await discord_message.add_reaction('🔄')
else:
log.warning("Skipping Discord mutations, DISCORD_MUTATIONS_ENABLED not set")
# DIGEST
week_ago_dt = datetime.utcnow() - timedelta(weeks=1)
with db:
last_digest_message = Message.last_bot_message(DIGEST_CHANNEL, '🔥')
if last_digest_message:
since_dt = last_digest_message.created_at
log.info(f"Last digest on {since_dt}")
if since_dt.date() > week_ago_dt.date():
log.info(f"Aborting, {since_dt.date()} (last digest) > {week_ago_dt.date()} (week ago)")
return # abort
else:
log.info(f"About to create digest, {since_dt.date()} (last digest) <= {week_ago_dt.date()} (week ago)")
else:
since_dt = week_ago_dt
log.info(f"Last digest not found, analyzing since {week_ago_dt}")
channel = await client.fetch_channel(DIGEST_CHANNEL)
with db:
messages = Message.digest_listing(since_dt, limit=DIGEST_LIMIT)
for n, message in enumerate(messages, start=1):
log.info(f"Digest #{n}: {message.upvotes} votes for {message.author.display_name} in #{message.channel_name}, {message.url}")
if DISCORD_MUTATIONS_ENABLED:
content = [
f"🔥 **{DIGEST_LIMIT} nej pÅ™ÃspÄ›vků za uplynulý týden (od {since_dt.day}.{since_dt.month}.)**",
"",
"Pokud je nÄ›co zajÃmavé nebo ti to pomohlo, dej tomu palec ðŸ‘�, srdÃÄ�ko â�¤ï¸�, oÄ�iÄ�ka 👀, apod. OcenÃÅ¡ autory a pomůžeÅ¡ tomu, aby se pÅ™ÃspÄ›vek mohl objevit i tady. NÄ›komu, kdo nemá Ä�as procházet vÅ¡echno, co se v klubu napÃÅ¡e, se může tento pÅ™ehled hodit.",
]
embed_description = []
for message in messages:
embed_description += [
f"{message.upvotes}× láska pro {message.author.mention} v {message.channel_mention}:",
f"> {textwrap.shorten(message.content, 200, placeholder='…')}",
f"[Hop na pÅ™ÃspÄ›vek]({message.url})",
"",
]
await channel.send(content="\n".join(content),
embed=Embed(description="\n".join(embed_description)))
else:
log.warning("Skipping Discord mutations, DISCORD_MUTATIONS_ENABLED not set")
async def main(client):
with db:
db.drop_tables([ClubMessage, ClubUser, ClubPinReaction])
db.create_tables([ClubMessage, ClubUser, ClubPinReaction])
authors = {}
relevant_channels = (channel for channel in client.juniorguru_guild.text_channels
if channel.permissions_for(client.juniorguru_guild.me).read_messages)
for channel in relevant_channels:
log.info(f'Channel #{channel.name}')
async for message in channel.history(limit=None, after=None):
if message.author.id not in authors:
# The message.author can be an instance of Member, but it can also be an instance of User,
# if the author isn't a member of the Discord guild/server anymore. User instances don't
# have certain properties, hence the getattr() calls below.
with db:
log.info(f"User '{message.author.display_name}' #{message.author.id}")
author = ClubUser.create(id=message.author.id,
is_bot=message.author.bot,
is_member=bool(getattr(message.author, 'joined_at', False)),
display_name=message.author.display_name,
mention=message.author.mention,
joined_at=getattr(message.author, 'joined_at', None),
roles=get_roles(message.author))
authors[message.author.id] = author
with db:
ClubMessage.create(id=message.id,
url=message.jump_url,
content=message.content,
upvotes_count=count_upvotes(message.reactions),
pin_reactions_count=count_pins(message.reactions),
created_at=message.created_at,
edited_at=message.edited_at,
author=authors[message.author.id],
channel_id=channel.id,
channel_name=channel.name,
channel_mention=channel.mention,
type=message.type.name)
users = set()
for reaction in message.reactions:
if emoji_name(reaction.emoji) in EMOJI_PINS:
for user in [user async for user in reaction.users()]:
users.add(user)
for user in users:
log.info(f"Message {message.jump_url} is pinned by user '{user.display_name}' #{user.id}")
ClubPinReaction.create(user=user.id, message=message.id)
# remaining members (did not author a single message)
log.info('Looking for remaining members, if any')
remaining_members = [member async for member in client.juniorguru_guild.fetch_members(limit=None)
if member.id not in authors]
for member in remaining_members:
with db:
log.info(f"Member '{member.display_name}' #{member.id}")
ClubUser.create(id=member.id,
is_bot=member.bot,
is_member=True,
display_name=member.display_name,
mention=member.mention,
joined_at=member.joined_at,
roles=get_roles(member))
with db:
messages_count = ClubMessage.count()
members_count = ClubUser.members_count()
log.info(f'Saved {messages_count} messages from {len(authors)} authors')
log.info(f'Saved {members_count} members')