from juniorguru.lib.log import get_log
from juniorguru.lib.club import discord_task, DISCORD_MUTATIONS_ENABLED
from juniorguru.models import ClubMessage
log = get_log('returning_members')
SYSTEM_MESSAGES_CHANNEL = 788823881024405544
@discord_task
async def main(client):
system_messages_channel = await client.fetch_channel(
SYSTEM_MESSAGES_CHANNEL)
for message in ClubMessage.channel_listing(SYSTEM_MESSAGES_CHANNEL):
if message.type == 'new_member' and message.author.first_seen_on(
) < message.created_at.date():
log.info(
f'It looks like {message.author.display_name} has returned')
discord_message = await system_messages_channel.fetch_message(
message.id)
if DISCORD_MUTATIONS_ENABLED:
await discord_message.add_reaction('👋')
await discord_message.add_reaction('🔄')
else:
log.warning(
"Skipping Discord mutations, DISCORD_MUTATIONS_ENABLED not set"
)
if __name__ == '__main__':
main()
from urllib.parse import urlparse
from pathlib import Path
from io import BytesIO
from PIL import Image
from juniorguru.lib.log import get_log
from juniorguru.lib.club import discord_task, is_default_avatar
from juniorguru.models import Member, db
log = get_log('members')
IMAGES_PATH = Path(__file__).parent.parent / 'images'
AVATARS_PATH = IMAGES_PATH / 'avatars'
SIZE_PX = 60
@discord_task
async def main(client):
AVATARS_PATH.mkdir(exist_ok=True, parents=True)
for path in AVATARS_PATH.glob('*.png'):
path.unlink()
with db:
Member.drop_table()
Member.create_table()
members = [member async for member in client.juniorguru_guild.fetch_members(limit=None)
if not member.bot]
from collections import Counter
from juniorguru.lib.log import get_log
from juniorguru.lib.club import discord_task, DISCORD_MUTATIONS_ENABLED, get_roles
from juniorguru.models import MessageAuthor, Event, db
log = get_log('roles')
ROLE_MOST_DISCUSSING = 836929320706113567
ROLE_MOST_HELPFUL = 836960665578766396
ROLE_IS_SPEAKER = 836928169092710441
ROLE_HAS_INTRO_AND_AVATAR = 836959652100702248
ROLE_IS_NEW = 836930259982352435
ROLE_IS_SPONSOR = 837316268142493736 # TODO
def main():
with db:
members = MessageAuthor.members_listing()
changes = []
top_members_limit = MessageAuthor.top_members_limit()
log.info(
f'members_count={len(members)}, top_members_limit={top_members_limit}'
)
# ROLE_MOST_DISCUSSING
messages_count_stats = calc_stats(members,
lambda m: m.messages_count(),
top_members_limit)
log.info(f"messages_count {repr_stats(members, messages_count_stats)}")
import random
from scrapy.downloadermiddlewares.retry import RetryMiddleware
from juniorguru.lib.log import get_log
from juniorguru.models import Proxy, db
log = get_log(__name__)
USER_AGENTS = [
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:81.0) Gecko/20100101 Firefox/81.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:75.0) Gecko/20100101 Firefox/75.0',
'Mozilla/5.0 (iPhone; CPU OS 14_0_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/29.0 Mobile/15E148 Safari/605.1.15',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15',
]
class ScrapingProxyMiddleware():
EXCEPTIONS_TO_RETRY = RetryMiddleware.EXCEPTIONS_TO_RETRY
@classmethod
def from_crawler(cls, crawler):
with db:
proxies = [proxy.address for proxy in Proxy.listing()]
return cls(proxies, crawler.settings)
def __init__(self, proxies, settings):
self.proxies = proxies
def get_proxy(self):
return random.choice(self.proxies[:5]) if self.proxies else None
import re
from collections import Counter
from juniorguru.lib.log import get_log
from juniorguru.models import ClubMessage, Topic, db
log = get_log('topics')
KEYWORDS = {
re.compile(r'\b' + key + r'\b', re.IGNORECASE): value
for key, value in {
r'pyladies|pylady': 'pyladies',
r'cs50': 'cs50',
r'enget\w+': 'engeto',
r'czechitas': 'czechitas',
r'(datov\w+|digit\w+) akademi\w+': 'czechitas',
r'it[ \-]?network': 'itnetwork',
r'pohovor\w*': 'interviews',
r'react ?girls': 'reactgirls',
r'python\w*': 'python',
r'djang\w+': 'django',
r'php': 'php',
r'p[ée]h[aá]pk\w+': 'php',
r'\w*sql\w*': 'sql',
r'nette': 'nette',
r'sym(f|ph)ony': 'symfony',
r'laravel\w*': 'laravel',
r'js': 'javascript',
r'javascript\w*': 'javascript',
r'flask\w*': 'flask',
r'react\w*': 'react',
from juniorguru.sync.metrics import main as sync_metrics
from juniorguru.sync.stories import main as sync_stories
from juniorguru.sync.supporters import main as sync_supporters
from juniorguru.sync.last_modified import main as sync_last_modified
from juniorguru.sync.press_releases import main as sync_press_releases
from juniorguru.sync.newsletter_mentions import main as sync_newsletter_mentions
from juniorguru.sync.transactions import main as sync_transactions
from juniorguru.sync.proxies import main as sync_proxies
from juniorguru.sync.members import main as sync_members
from juniorguru.sync.events import main as sync_events
from juniorguru.sync.messages import main as sync_messages
from juniorguru.sync.topics import main as sync_topics
from juniorguru.sync.roles import main as sync_roles
from juniorguru.lib.magic import do_magic
log = get_log('sync')
@timer.notify
def main():
# order-insensitive
sync_stories()
sync_supporters()
sync_last_modified()
sync_press_releases()
sync_logos()
sync_transactions()
sync_proxies()
sync_members()
sync_messages()
import subprocess
from multiprocessing import Pool
from pathlib import Path
from juniorguru.lib.log import get_log
from juniorguru.lib.club import discord_task, count_downvotes, count_upvotes, DISCORD_MUTATIONS_ENABLED
from juniorguru.models import Job, JobDropped, JobError, SpiderMetric, db
from juniorguru.scrapers.settings import IMAGES_STORE
log = get_log('jobs')
JOBS_CHANNEL = 834443926655598592
JOBS_VOTING_CHANNEL = 841680291675242546
def main():
# If the creation of the directory is left to the spiders, they can end
# up colliding in making sure it gets created
Path(IMAGES_STORE).mkdir(exist_ok=True, parents=True)
with db:
db.drop_tables([Job, JobError, JobDropped, SpiderMetric])
db.create_tables([Job, JobError, JobDropped, SpiderMetric])
spider_names = [
'juniorguru',
'linkedin',
'stackoverflow',
'startupjobs',
'remoteok',
'wwr',
import os
from pathlib import Path
from datetime import date, timedelta
import arrow
from playhouse.shortcuts import model_to_dict
from strictyaml import Datetime, Map, Seq, Str, Url, Int, Optional, load
from juniorguru.models import Event, EventSpeaking, Message, db
from juniorguru.lib.images import render_image_file, downsize_square_photo, save_as_ig_square
from juniorguru.lib.log import get_log
from juniorguru.lib.md import strip_links
from juniorguru.lib.template_filters import local_time, md, weekday
from juniorguru.lib.club import DISCORD_MUTATIONS_ENABLED, discord_task
log = get_log('events')
FLUSH_POSTERS = bool(int(os.getenv('FLUSH_POSTERS', 0)))
DATA_DIR = Path(__file__).parent.parent / 'data'
IMAGES_DIR = Path(__file__).parent.parent / 'images'
POSTERS_DIR = IMAGES_DIR / 'posters'
ANNOUNCEMENTS_CHANNEL = 789046675247333397
EVENTS_CHAT_CHANNEL = 821411678167367691
schema = Seq(
Map({
'title': Str(),
'date': Datetime(),
Optional('time', default='18:00'): Str(),
'description': Str(),
import time
import json
from pathlib import Path
from collections.abc import Set
from functools import wraps
import scrapy
from peewee import Model, SqliteDatabase, OperationalError
from playhouse.sqlite_ext import JSONField as BaseJSONField
from juniorguru.lib.log import get_log
log = get_log('db')
db_file = Path(__file__).parent / '..' / 'data' / 'data.db'
db = SqliteDatabase(db_file, check_same_thread=False,
pragmas={'journal_mode': 'wal'})
class BaseModel(Model):
class Meta:
database = db
class JSONField(BaseJSONField):
def __init__(self, *args, **kwargs):
kwargs.setdefault('json_dumps', json_dumps)
super().__init__(*args, **kwargs)
from datetime import date
import os
import sys
import smtplib
import random
from email.message import EmailMessage
from email.headerregistry import Address
from juniorguru.lib.log import get_log
from juniorguru.send import job_metrics, logo_metrics
log = get_log('send')
EMAIL_BATCHES = (
(job_metrics, 'Monday?', lambda date: date.weekday() == 0),
(logo_metrics, 'First day of the month?', lambda date: date.today().day == 1),
)
def main():
config = os.environ
debug = os.getenv('DEBUG', '--debug' in sys.argv)
log.info(f"Debug: {'YES' if debug else 'NO'}")
today = date.today()
log.info(f"Today: {today:%Y-%m-%d}")
for module, when_label, when in EMAIL_BATCHES:
log.info(f"About to send {module.__name__}")
if when(today):
import os
from multiprocessing import Pool
import random
import requests
from lxml import html
from juniorguru.models import Proxy, db
from juniorguru.lib.log import get_log
log = get_log('proxies')
def main():
proxies = []
if os.getenv('PROXIES_ENABLED'):
response = requests.get(
'https://free-proxy-list.net/',
headers={
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.8,cs;q=0.6,sk;q=0.4,es;q=0.2',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:81.0) Gecko/20100101 Firefox/81.0',
'Referer': 'https://www.sslproxies.org/',
})
response.raise_for_status()
html_tree = html.fromstring(response.text)
rows = iter(html_tree.cssselect('#proxylisttable tr'))
headers = [col.text_content() for col in next(rows)]
for row in rows:
from pathlib import Path
from urllib.parse import urlparse
from io import BytesIO
from PIL import Image
from juniorguru.lib.log import get_log
from juniorguru.lib.club import discord_task, is_default_avatar
from juniorguru.models import ClubUser, db
log = get_log('avatars')
IMAGES_PATH = Path(__file__).parent.parent / 'images'
AVATARS_PATH = IMAGES_PATH / 'avatars'
AVATAR_SIZE_PX = 60
@discord_task
async def main(client):
AVATARS_PATH.mkdir(exist_ok=True, parents=True)
for path in AVATARS_PATH.glob('*.png'):
path.unlink()
with db:
for member in ClubUser.members_listing():
log.info(
f"Downloading avatar for '{member.display_name}' #{member.id}")
discord_member = await client.juniorguru_guild.fetch_member(
member.id)
member.avatar_path = await download_avatar(discord_member)
log.info(f"Result: '{member.avatar_path}'")
import textwrap
from discord import Embed
from juniorguru.lib.log import get_log
from juniorguru.lib.club import discord_task, DISCORD_MUTATIONS_ENABLED
from juniorguru.models import ClubPinReaction, ClubUser, ClubMessage, db
log = get_log('pins')
@discord_task
async def main(client):
with db:
top_members_limit = ClubUser.top_members_limit()
pin_reactions_by_members = [
pin_reaction for pin_reaction in ClubPinReaction.listing()
if pin_reaction.user.is_member
]
log.info(
f'Found {len(pin_reactions_by_members)} pin reactions by people who are currently members'
)
for pin_reaction in pin_reactions_by_members:
member = await client.juniorguru_guild.fetch_member(
pin_reaction.user.id)
if member.dm_channel:
channel = member.dm_channel
else:
channel = await member.create_dm()
import os
from gql import Client as Memberful, gql
from gql.transport.requests import RequestsHTTPTransport
# import stripe
from juniorguru.lib.log import get_log
log = get_log('subscriptions')
def main():
# stripe.api_key = os.environ['STRIPE_API_KEY']
# for subscription in stripe.Subscription.list().auto_paging_iter():
# print(subscription['metadata'])
# https://memberful.com/help/integrate/advanced/memberful-api/
# https://juniorguru.memberful.com/api/graphql/explorer
api_key = os.environ['MEMBERFUL_API_KEY']
transport = RequestsHTTPTransport(url='https://juniorguru.memberful.com/api/graphql/',
headers={'Authorization': f'Bearer {api_key}'},
verify=True, retries=3)
memberful = Memberful(transport=transport, fetch_schema_from_transport=True)
query = gql("""
query getSubscriptions($cursor: String!) {
subscriptions(after: $cursor) {
totalCount
pageInfo {
endCursor
import os
from pathlib import Path
import arrow
from flask import Flask, Response, render_template, url_for
from juniorguru.lib.log import get_log
from juniorguru.lib import template_filters
from juniorguru.lib.images import render_image_file
from juniorguru.models import Job, Metric, Story, Supporter, LastModified, PressRelease, Logo, Member, db
log = get_log('web')
FLUSH_THUMBNAILS = bool(int(os.getenv('FLUSH_THUMBNAILS', 0)))
THUMBNAILS_DIR = Path(__file__).parent / 'static' / 'images' / 'thumbnails'
NAV_TABS = [
{
'endpoint': 'motivation',
'name': 'Příručka'
},
{
'endpoint': 'jobs',
'name': 'Práce'
},
{
'endpoint': 'club',
'name': 'Klub'
},
]
import textwrap
from datetime import datetime, timedelta
from discord import Embed
from juniorguru.lib.log import get_log
from juniorguru.lib.club import discord_task, count_upvotes, is_default_avatar, get_roles, DISCORD_MUTATIONS_ENABLED
from juniorguru.models import Message, MessageAuthor, db
log = get_log('messages')
DIGEST_CHANNEL = 789046675247333397
SYSTEM_MESSAGES_CHANNEL = 788823881024405544
DIGEST_LIMIT = 5
@discord_task
async def main(client):
# MESSAGES AND AUTHORS
with db:
db.drop_tables([Message, MessageAuthor])
db.create_tables([Message, MessageAuthor])
authors = {}
relevant_channels = (channel for channel in client.juniorguru_guild.text_channels
if channel.permissions_for(client.juniorguru_guild.me).read_messages)
for channel in relevant_channels:
log.info(f'#{channel.name}')
async for message in channel.history(limit=None, after=None):
import textwrap
from datetime import datetime, timedelta
from discord import Embed
from juniorguru.lib.log import get_log
from juniorguru.lib.club import discord_task, DISCORD_MUTATIONS_ENABLED
from juniorguru.models import ClubMessage, db
log = get_log('digest')
DIGEST_CHANNEL = 789046675247333397
DIGEST_LIMIT = 5
@discord_task
async def main(client):
week_ago_dt = datetime.utcnow() - timedelta(weeks=1)
with db:
last_digest_message = ClubMessage.last_bot_message(DIGEST_CHANNEL, '🔥')
if last_digest_message:
since_dt = last_digest_message.created_at
log.info(f"Last digest on {since_dt}")
if since_dt.date() > week_ago_dt.date():
log.info(f"Aborting, {since_dt.date()} (last digest) > {week_ago_dt.date()} (week ago)")
return # abort
else:
log.info(f"About to create digest, {since_dt.date()} (last digest) <= {week_ago_dt.date()} (week ago)")
else:
from juniorguru.lib.log import get_log
from juniorguru.lib.club import EMOJI_PINS, discord_task, count_upvotes, emoji_name, get_roles, count_pins
from juniorguru.models import ClubMessage, ClubUser, ClubPinReaction, db
log = get_log('club_content')
@discord_task
async def main(client):
with db:
db.drop_tables([ClubMessage, ClubUser, ClubPinReaction])
db.create_tables([ClubMessage, ClubUser, ClubPinReaction])
authors = {}
relevant_channels = (channel for channel in client.juniorguru_guild.text_channels
if channel.permissions_for(client.juniorguru_guild.me).read_messages)
for channel in relevant_channels:
log.info(f'Channel #{channel.name}')
async for message in channel.history(limit=None, after=None):
if message.author.id not in authors:
# The message.author can be an instance of Member, but it can also be an instance of User,
# if the author isn't a member of the Discord guild/server anymore. User instances don't
# have certain properties, hence the getattr() calls below.
with db:
log.info(f"User '{message.author.display_name}' #{message.author.id}")
author = ClubUser.create(id=message.author.id,
is_bot=message.author.bot,
is_member=bool(getattr(message.author, 'joined_at', False)),
display_name=message.author.display_name,
mention=message.author.mention,
