def main(): # If the creation of the directory is left to the spiders, they can end # up colliding in making sure it gets created Path(IMAGES_STORE).mkdir(exist_ok=True, parents=True) with db: db.drop_tables([Job, JobError, JobDropped, SpiderMetric]) db.create_tables([Job, JobError, JobDropped, SpiderMetric]) spider_names = [ 'juniorguru', 'linkedin', 'stackoverflow', 'startupjobs', 'remoteok', 'wwr', 'dobrysef', ] Pool().map(run_spider, spider_names) manage_jobs_channel()
def main(): path = DATA_DIR / 'events.yml' records = [ load_record(record.data) for record in load(path.read_text(), schema) ] if FLUSH_POSTERS: log.warning("Removing all existing posters, FLUSH_POSTERS is set") for poster_path in POSTERS_DIR.glob('*.png'): poster_path.unlink() with db: db.drop_tables([Event, EventSpeaking]) db.create_tables([Event, EventSpeaking]) # process data from the YAML, generate posters for record in records: name = record['title'] log.info(f"Creating '{name}'") speakers_ids = record.pop('speakers', []) event = Event.create(**record) for speaker_id in speakers_ids: try: avatar_path = next( (IMAGES_DIR / 'speakers').glob(f"{speaker_id}.*")) except StopIteration: log.info(f"Didn't find speaker avatar for {speaker_id}") avatar_path = None else: log.info(f"Downsizing speaker avatar for {speaker_id}") avatar_path = downsize_square_photo( avatar_path, 500).relative_to(IMAGES_DIR) log.info(f"Marking member {speaker_id} as a speaker") EventSpeaking.create(speaker=speaker_id, event=event, avatar_path=avatar_path) if event.logo_path: log.info(f"Checking '{event.logo_path}'") image_path = IMAGES_DIR / event.logo_path if not image_path.exists(): raise ValueError( f"Event '{name}' references '{image_path}', but it doesn't exist" ) log.info(f"Rendering poster for '{name}'") context = dict( event=model_to_dict(event, extra_attrs=['first_avatar_path'])) image_path = render_image_file('poster.html', context, POSTERS_DIR, filters={ 'md': md, 'local_time': local_time, 'weekday': weekday, }) event.poster_path = image_path.relative_to(IMAGES_DIR) event.save() log.info(f"Rendering Instagram poster for '{name}'") save_as_ig_square(image_path) # discord messages if DISCORD_MUTATIONS_ENABLED: post_next_event_messages() else: log.warning( "Skipping Discord mutations, DISCORD_MUTATIONS_ENABLED not set" )
async def main(client): # MESSAGES AND AUTHORS with db: db.drop_tables([Message, MessageAuthor]) db.create_tables([Message, MessageAuthor]) authors = {} relevant_channels = (channel for channel in client.juniorguru_guild.text_channels if channel.permissions_for(client.juniorguru_guild.me).read_messages) for channel in relevant_channels: log.info(f'#{channel.name}') async for message in channel.history(limit=None, after=None): if message.author.id not in authors: # The message.author can be an instance of Member, but it can also be an instance of User, # if the author isn't a member of the Discord guild/server anymore. User instances don't # have certain properties, hence the getattr() calls below. with db: author = MessageAuthor.create(id=message.author.id, is_bot=message.author.bot, is_member=bool(getattr(message.author, 'joined_at', False)), has_avatar=not is_default_avatar(message.author.avatar_url), display_name=message.author.display_name, mention=message.author.mention, joined_at=getattr(message.author, 'joined_at', None), roles=get_roles(message.author)) authors[message.author.id] = author with db: Message.create(id=message.id, url=message.jump_url, content=message.content, upvotes=count_upvotes(message.reactions), created_at=message.created_at, edited_at=message.edited_at, author=authors[message.author.id], channel_id=channel.id, channel_name=channel.name, channel_mention=channel.mention, type=message.type.name) with db: messages_count = Message.count() log.info(f'Saved {messages_count} messages from {len(authors)} authors') # RETURNING MEMBERS system_messages_channel = await client.fetch_channel(SYSTEM_MESSAGES_CHANNEL) for message in Message.channel_listing(SYSTEM_MESSAGES_CHANNEL): if message.type == 'new_member' and message.author.first_seen_at() < message.created_at.date(): log.info(f'It looks like {message.author.display_name} has returned') discord_message = await system_messages_channel.fetch_message(message.id) if DISCORD_MUTATIONS_ENABLED: await discord_message.add_reaction('👋') await discord_message.add_reaction('🔄') else: log.warning("Skipping Discord mutations, DISCORD_MUTATIONS_ENABLED not set") # DIGEST week_ago_dt = datetime.utcnow() - timedelta(weeks=1) with db: last_digest_message = Message.last_bot_message(DIGEST_CHANNEL, '🔥') if last_digest_message: since_dt = last_digest_message.created_at log.info(f"Last digest on {since_dt}") if since_dt.date() > week_ago_dt.date(): log.info(f"Aborting, {since_dt.date()} (last digest) > {week_ago_dt.date()} (week ago)") return # abort else: log.info(f"About to create digest, {since_dt.date()} (last digest) <= {week_ago_dt.date()} (week ago)") else: since_dt = week_ago_dt log.info(f"Last digest not found, analyzing since {week_ago_dt}") channel = await client.fetch_channel(DIGEST_CHANNEL) with db: messages = Message.digest_listing(since_dt, limit=DIGEST_LIMIT) for n, message in enumerate(messages, start=1): log.info(f"Digest #{n}: {message.upvotes} votes for {message.author.display_name} in #{message.channel_name}, {message.url}") if DISCORD_MUTATIONS_ENABLED: content = [ f"🔥 **{DIGEST_LIMIT} nej pÅ™ÃspÄ›vků za uplynulý týden (od {since_dt.day}.{since_dt.month}.)**", "", "Pokud je nÄ›co zajÃmavé nebo ti to pomohlo, dej tomu palec ðŸ‘�, srdÃÄ�ko â�¤ï¸�, oÄ�iÄ�ka 👀, apod. OcenÃÅ¡ autory a pomůžeÅ¡ tomu, aby se pÅ™ÃspÄ›vek mohl objevit i tady. NÄ›komu, kdo nemá Ä�as procházet vÅ¡echno, co se v klubu napÃÅ¡e, se může tento pÅ™ehled hodit.", ] embed_description = [] for message in messages: embed_description += [ f"{message.upvotes}× láska pro {message.author.mention} v {message.channel_mention}:", f"> {textwrap.shorten(message.content, 200, placeholder='…')}", f"[Hop na pÅ™ÃspÄ›vek]({message.url})", "", ] await channel.send(content="\n".join(content), embed=Embed(description="\n".join(embed_description))) else: log.warning("Skipping Discord mutations, DISCORD_MUTATIONS_ENABLED not set")
async def main(client): with db: db.drop_tables([ClubMessage, ClubUser, ClubPinReaction]) db.create_tables([ClubMessage, ClubUser, ClubPinReaction]) authors = {} relevant_channels = (channel for channel in client.juniorguru_guild.text_channels if channel.permissions_for(client.juniorguru_guild.me).read_messages) for channel in relevant_channels: log.info(f'Channel #{channel.name}') async for message in channel.history(limit=None, after=None): if message.author.id not in authors: # The message.author can be an instance of Member, but it can also be an instance of User, # if the author isn't a member of the Discord guild/server anymore. User instances don't # have certain properties, hence the getattr() calls below. with db: log.info(f"User '{message.author.display_name}' #{message.author.id}") author = ClubUser.create(id=message.author.id, is_bot=message.author.bot, is_member=bool(getattr(message.author, 'joined_at', False)), display_name=message.author.display_name, mention=message.author.mention, joined_at=getattr(message.author, 'joined_at', None), roles=get_roles(message.author)) authors[message.author.id] = author with db: ClubMessage.create(id=message.id, url=message.jump_url, content=message.content, upvotes_count=count_upvotes(message.reactions), pin_reactions_count=count_pins(message.reactions), created_at=message.created_at, edited_at=message.edited_at, author=authors[message.author.id], channel_id=channel.id, channel_name=channel.name, channel_mention=channel.mention, type=message.type.name) users = set() for reaction in message.reactions: if emoji_name(reaction.emoji) in EMOJI_PINS: for user in [user async for user in reaction.users()]: users.add(user) for user in users: log.info(f"Message {message.jump_url} is pinned by user '{user.display_name}' #{user.id}") ClubPinReaction.create(user=user.id, message=message.id) # remaining members (did not author a single message) log.info('Looking for remaining members, if any') remaining_members = [member async for member in client.juniorguru_guild.fetch_members(limit=None) if member.id not in authors] for member in remaining_members: with db: log.info(f"Member '{member.display_name}' #{member.id}") ClubUser.create(id=member.id, is_bot=member.bot, is_member=True, display_name=member.display_name, mention=member.mention, joined_at=member.joined_at, roles=get_roles(member)) with db: messages_count = ClubMessage.count() members_count = ClubUser.members_count() log.info(f'Saved {messages_count} messages from {len(authors)} authors') log.info(f'Saved {members_count} members')