def __init__(self) -> None: if Mailer.__instance is not None: raise Exception("This class is a singleton!") Mailer.__instance = self self.init_ok: bool = False self.msg: EmailMessage = None self.smtp_host: str = Config.get('MAILER', 'smtp_host') self.smtp_port: str = Config.get('MAILER', 'smtp_port') self.smtp_user: str = Config.get('MAILER', 'smtp_user') self.smtp_password: str = Config.get('MAILER', 'smtp_password') _sender = Config.get('MAILER', 'sender') try: _receivers = Config.get('MAILER', 'receivers', param_type="json") except JSONDecodeError: Logger.get(self.__class__.__name__).warn('Problems decoding email receivers from JSON in config!') _receivers = None if not _sender or not _receivers or not self.smtp_host or not self.smtp_port: Logger.get(self.__class__.__name__).warn('Emails not being sent!') return self.msg = EmailMessage() self.msg.set_charset('utf-8') self.msg['From'] = _sender self.msg['To'] = _receivers self.init_ok = True
def __init__(self) -> None: self.url = Config.get('URL_FETCHER', 'url') try: self.headers = Config.get('URL_FETCHER', 'headers', param_type="json") except JSONDecodeError: Logger.get(self.__class__.__name__).warn( "Problem decoding URL headers from JSON in config!") self.headers = None self.content: Optional[str] = None
def fetch(self) -> None: # noinspection PyBroadException try: response = requests.get(self.url, headers=self.headers) except Exception as e: Logger.get(self.__class__.__name__).error(str(e)) self.content = None return if response.status_code >= 300: Logger.get(self.__class__.__name__).error('{}: {}'.format( response.status_code, response.reason)) self.content = None else: self.content = response.content.decode('utf-8')
def send(cls, content: AbstractMailTemplate, subject: str = None) -> None: if Mailer.__instance is None: Mailer() if Mailer.__instance.init_ok: smtp = smtplib.SMTP_SSL("{}:{}".format(Mailer.__instance.smtp_host, Mailer.__instance.smtp_port)) try: smtp.login(Mailer.__instance.smtp_user, Mailer.__instance.smtp_password) except smtplib.SMTPAuthenticationError as e: Logger.get(cls.__name__).warn(str(e)) return Mailer.__instance.msg['Subject'] = subject if subject is not None else Config.get('MAILER', 'Subject') Mailer.__instance.msg.set_content(content.get()) try: smtp.send_message(Mailer.__instance.msg) except smtplib.SMTPDataError as e: Logger.get(cls.__name__).warn(str(e)) smtp.quit()
def parse(self) -> None: contents = self.html[self.html.index(Config.get('PARSER', 'start_tag') ) + len(Config.get('PARSER', 'start_tag')):self.html. index(Config.get('PARSER', 'end_tag'))].strip() if Config.get('PARSER', 'empty_pattern') in contents: Logger.get(self.__class__.__name__).info("No new contents.") self.has_new_content = False self.links = {} try: os.remove("{}/{}".format(Config.get('CRON', 'project_root'), Config.get('PARSER', 'cache_file'))) except OSError as e: if e.errno != errno.ENOENT: raise return with open(Config.get('PARSER', 'cache_file'), 'a+') as f: f.seek(0) if f.read() == contents: Logger.get(self.__class__.__name__).info("No new contents.") self.has_new_content = False self.links = {} else: Logger.get( self.__class__.__name__).info("New content detected.") self.has_new_content = True self._extract_links(contents) f.seek(0) f.truncate() f.write(contents)
def __init__(self, session): self.session = session self.logger = Logger.get(__name__)
class App: DEBUG = True if DEBUG: CONFIG_PATH = "config_testing.json" else: CONFIG_PATH = "config.json" with open(CONFIG_PATH) as f: CONFIG = json.load(f) logger = Logger.get(__name__) def main(self): if not database_exists(self.CONFIG["database_url"]): self.logger.info("Creating new database") Database.init_db(self.CONFIG["database_url"]) self.logger.info("Connecting to the database") db = Database.from_url(self.CONFIG["database_url"]) users = self.update_users_preferences(db) venues = db.get_distinctive_items("venue") artists = db.get_distinctive_items("artist") promoters = db.get_distinctive_items("promoter") # go through venues new_events = [] for venue in venues: self.logger.info(f"Checking {venue['name']} venue...") venue["type"] = "venue" venue_events = self.get_events( venue, self.CONFIG["venue_url_prefix"] + venue["tag"] ) new_venue_events = self.add_to_database(db, venue_events) new_events.extend(new_venue_events) for artist in artists: self.logger.info(f"Checking {artist['name']} artist...") artist["type"] = "artist" artist_events = self.get_events( artist, self.CONFIG["artist_url_prefix"] + artist["tag"] ) new_artist_events = self.add_to_database(db, artist_events) new_events.extend(new_artist_events) for promoter in promoters: self.logger.info(f"Checking {promoter['name']} promoter...") promoter["type"] = "promoter" promoter_events = self.get_events( promoter, self.CONFIG["promoter_url_prefix"] + promoter["tag"] ) new_promoter_events = self.add_to_database(db, promoter_events) new_events.extend(new_promoter_events) db.commit() for new_event in new_events: self.add_event_notifications(new_event, users) self.send_emails(users) def update_users_preferences(self, db): self.logger.info("Downloading users favourites") users = self.get_users() users = self.download_users_interests(users) self.logger.info("Updating users database") self.update_database(users, db) return users def get_users(self): with open(self.CONFIG["users_path"]) as f: users_json = json.load(f) users = [] for user_data in users_json["users"]: users.append( User( user_data["name"], user_data["nickname"], user_data["email"], user_data["locations"], ) ) return users def download_users_interests(self, users): with requests.Session() as s: login_response = s.post( self.CONFIG["login_url"], data=self.CONFIG["payload"], headers=self.CONFIG["headers"], ) if not self.logged_in(login_response): raise Exception("Could not login to Resident Advisor") for user in users: self.logger.info(f"Fetching user {user.nickname} preferences") url = self.CONFIG["profile_url_prefix"] + user.nickname + "/favourites" html = s.get(url, headers=self.CONFIG["headers"]) html.encoding = "utf-8" soup = BeautifulSoup(html.text, "html.parser") html_artists = soup.find_all("div", class_="fav") if html_artists is not None: for artist in html_artists: info_tag = artist.find("div", class_="pb2").find("a") artist_name = info_tag.get_text() artist_tag = info_tag.get("href")[4:] user.add_artist(artist_name, artist_tag) else: self.logger.warning( f"User {user.nickname} does not follow any artists" ) html_venues = soup.find("ul", class_="list venueListing") if html_venues is not None: for venue in html_venues.find_all("li", recursive=False): info_tag = venue.find_all("a")[1] venue_name = info_tag.get_text() venue_tag = info_tag.get("href")[14:] user.add_venue(venue_name, venue_tag) else: self.logger.warning( f"User {user.nickname} does not follow any venues" ) try: try: html_promoters = soup.find_all( lambda tag: tag.name == "ul" and tag.get("class") == ["list"] )[1] except Exception: # if find_all doesn't succeed, it means the person does not # follow any labels and only promoters are available html_promoters = soup.find( lambda tag: tag.name == "ul" and tag.get("class") == ["list"] ) for promoter in html_promoters.find_all("li", recursive=False): info_tag = promoter.find_all("a")[1] promoter_name = info_tag.get_text() promoter_tag = info_tag.get("href")[18:] user.add_promoter(promoter_name, promoter_tag) except Exception: self.logger.warning( f"User {user.nickname} does not follow any promoters" ) return users def logged_in(self, login_response): login_response.encoding = "utf-8" soup = BeautifulSoup(login_response.text, "html.parser") spans = soup.find_all("span") for span in spans: if span.get_text() == "Welcome": return True return False def update_database(self, users, db): for user in users: db.update_user(user) def get_events(self, entity, url): html = requests.get(url) html.encoding = "utf-8" soup = BeautifulSoup(html.text, "html.parser") events_html = soup.find_all("article", class_="event-item") events = [] for event_html in events_html: try: if entity["type"] is "venue": event = Event.from_venue_html(entity["name"], event_html) elif entity["type"] is "artist": event = Event.from_artist_html(entity["name"], event_html) elif entity["type"] is "promoter": event = Event.from_promoter_html(entity["name"], event_html) except: self.logger.warning( f"Could not generate event from the following html: {event_html.get_text()}" ) events.append(event) return events def add_to_database(self, db, events): new_events = [] for event in events: event_in_database = db.fetch_from_database(event.event_id, event.event_type) if event_in_database is not None and event_in_database.tickets_available: continue event.tickets = self.get_tickets(event.event_url) tickets_available = (True, False)[not event.tickets] # If event is not in db, add if event_in_database is None: db.add_event(event.event_id, event.event_type, tickets_available) self.logger.info( f"NEW EVENT WITH URL {event.event_url} IS ADDED TO THE DATABASE. TICKETS: {tickets_available}" ) new_events.append(event) # The event was in database but didn't have tickets else: if tickets_available: db.update_event(event.event_id, event.event_type, tickets_available) self.logger.info( f"EVENT WITH URL {event.event_url} WAS UPDATED WITH TICKETS" ) new_events.append(event) return new_events def get_tickets(self, event_url): html = requests.get(event_url) html.encoding = "utf-8" soup = BeautifulSoup(html.text, "html.parser") tickets = [] try: html_tickets = soup.find_all("li", class_="onsale but") for html_ticket in html_tickets: p = html_ticket.find("p") tickets.append( { "name": p.find(text=True, recursive=False), "price": p.find("span").get_text(), } ) except: self.logger.warning(f"some problem getting tickets for {event_url}") return tickets def add_event_notifications(self, event, users): for user in users: user.add_to_email(event) def send_emails(self, users): service = discovery.build("gmail", "v1", credentials=self.make_credentials()) for user in users: if user.number_of_new_events == 0: continue user.add_email_ending() self.logger.info(f"Emailing {user.email}") message = MIMEMultipart() message["From"] = "me" message["Subject"] = "New events on RA" message["To"] = user.email message.attach(MIMEText(user.email_body.get(), "html")) raw_message = base64.urlsafe_b64encode(message.as_bytes()).decode() body = {"raw": raw_message} self.send_email_request(service, body) return def make_credentials(self): with open(self.CONFIG["credentials_path"]) as f: data = json.load(f) return Credentials( None, refresh_token=data["refresh_token"], client_id=data["installed"]["client_id"], client_secret=data["installed"]["client_secret"], token_uri=data["installed"]["token_uri"], ) @backoff.on_exception(backoff.expo, HttpError, max_tries=4) def send_email_request(self, service, body): service.users().messages().send(userId="me", body=body).execute()